promptfoo 0.113.0 → 0.113.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +28 -30
- package/dist/src/app/assets/{index-BBF3o-At.js → index-DDgDcSbp.js} +176 -176
- package/dist/src/app/assets/{index.es-CIVyj4Ly.js → index.es-CTuWilQA.js} +1 -1
- package/dist/src/app/assets/{source-map-support-DmE-TbSd.js → source-map-support-CMD9DZ3t.js} +1 -1
- package/dist/src/app/assets/{sync-DKyeSqmb.js → sync-DXeGLZqh.js} +1 -1
- package/dist/src/app/index.html +1 -1
- package/dist/src/commands/auth.d.ts.map +1 -1
- package/dist/src/commands/auth.js +2 -0
- package/dist/src/commands/auth.js.map +1 -1
- package/dist/src/commands/cache.d.ts.map +1 -1
- package/dist/src/commands/cache.js +1 -0
- package/dist/src/commands/cache.js.map +1 -1
- package/dist/src/commands/config.d.ts.map +1 -1
- package/dist/src/commands/config.js +3 -0
- package/dist/src/commands/config.js.map +1 -1
- package/dist/src/commands/delete.d.ts.map +1 -1
- package/dist/src/commands/delete.js +1 -0
- package/dist/src/commands/delete.js.map +1 -1
- package/dist/src/commands/eval.d.ts.map +1 -1
- package/dist/src/commands/eval.js +2 -0
- package/dist/src/commands/eval.js.map +1 -1
- package/dist/src/commands/export.d.ts.map +1 -1
- package/dist/src/commands/export.js +1 -0
- package/dist/src/commands/export.js.map +1 -1
- package/dist/src/commands/generate/dataset.d.ts.map +1 -1
- package/dist/src/commands/generate/dataset.js +2 -0
- package/dist/src/commands/generate/dataset.js.map +1 -1
- package/dist/src/commands/import.d.ts.map +1 -1
- package/dist/src/commands/import.js +1 -0
- package/dist/src/commands/import.js.map +1 -1
- package/dist/src/commands/init.d.ts.map +1 -1
- package/dist/src/commands/init.js +1 -0
- package/dist/src/commands/init.js.map +1 -1
- package/dist/src/commands/list.d.ts.map +1 -1
- package/dist/src/commands/list.js +3 -0
- package/dist/src/commands/list.js.map +1 -1
- package/dist/src/commands/share.d.ts.map +1 -1
- package/dist/src/commands/share.js +3 -2
- package/dist/src/commands/share.js.map +1 -1
- package/dist/src/commands/show.d.ts.map +1 -1
- package/dist/src/commands/show.js +4 -0
- package/dist/src/commands/show.js.map +1 -1
- package/dist/src/commands/view.d.ts.map +1 -1
- package/dist/src/commands/view.js +1 -0
- package/dist/src/commands/view.js.map +1 -1
- package/dist/src/configTypes.d.ts +0 -1
- package/dist/src/configTypes.d.ts.map +1 -1
- package/dist/src/evaluator.js +1 -1
- package/dist/src/evaluator.js.map +1 -1
- package/dist/src/evaluatorHelpers.js +3 -3
- package/dist/src/evaluatorHelpers.js.map +1 -1
- package/dist/src/globalConfig/accounts.d.ts.map +1 -1
- package/dist/src/globalConfig/accounts.js +0 -1
- package/dist/src/globalConfig/accounts.js.map +1 -1
- package/dist/src/globalConfig/globalConfig.d.ts +1 -1
- package/dist/src/globalConfig/globalConfig.d.ts.map +1 -1
- package/dist/src/globalConfig/globalConfig.js +9 -17
- package/dist/src/globalConfig/globalConfig.js.map +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +2 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/main.js +1 -1
- package/dist/src/main.js.map +1 -1
- package/dist/src/onboarding.js +1 -1
- package/dist/src/onboarding.js.map +1 -1
- package/dist/src/providers/bam.d.ts +22 -16
- package/dist/src/providers/bam.d.ts.map +1 -1
- package/dist/src/providers/bedrock.js +1 -1
- package/dist/src/providers/bedrock.js.map +1 -1
- package/dist/src/providers/bedrockKnowledgeBase.js +1 -1
- package/dist/src/providers/bedrockKnowledgeBase.js.map +1 -1
- package/dist/src/providers/http.d.ts.map +1 -1
- package/dist/src/providers/http.js +17 -5
- package/dist/src/providers/http.js.map +1 -1
- package/dist/src/providers/sagemaker.js +1 -1
- package/dist/src/providers/sagemaker.js.map +1 -1
- package/dist/src/redteam/commands/discover.d.ts +117 -79
- package/dist/src/redteam/commands/discover.d.ts.map +1 -1
- package/dist/src/redteam/commands/discover.js +139 -179
- package/dist/src/redteam/commands/discover.js.map +1 -1
- package/dist/src/redteam/commands/generate.d.ts.map +1 -1
- package/dist/src/redteam/commands/generate.js +10 -11
- package/dist/src/redteam/commands/generate.js.map +1 -1
- package/dist/src/redteam/commands/init.js +1 -1
- package/dist/src/redteam/commands/init.js.map +1 -1
- package/dist/src/redteam/commands/poison.d.ts.map +1 -1
- package/dist/src/redteam/commands/poison.js +1 -0
- package/dist/src/redteam/commands/poison.js.map +1 -1
- package/dist/src/redteam/commands/report.d.ts.map +1 -1
- package/dist/src/redteam/commands/report.js +1 -0
- package/dist/src/redteam/commands/report.js.map +1 -1
- package/dist/src/redteam/commands/run.d.ts.map +1 -1
- package/dist/src/redteam/commands/run.js +1 -0
- package/dist/src/redteam/commands/run.js.map +1 -1
- package/dist/src/redteam/commands/setup.d.ts.map +1 -1
- package/dist/src/redteam/commands/setup.js +1 -0
- package/dist/src/redteam/commands/setup.js.map +1 -1
- package/dist/src/redteam/constants.d.ts +1 -1
- package/dist/src/redteam/constants.d.ts.map +1 -1
- package/dist/src/redteam/constants.js +1 -1
- package/dist/src/redteam/constants.js.map +1 -1
- package/dist/src/redteam/index.d.ts.map +1 -1
- package/dist/src/redteam/index.js +11 -9
- package/dist/src/redteam/index.js.map +1 -1
- package/dist/src/redteam/plugins/index.js +1 -1
- package/dist/src/redteam/plugins/index.js.map +1 -1
- package/dist/src/redteam/plugins/intent.d.ts +1 -1
- package/dist/src/redteam/plugins/intent.d.ts.map +1 -1
- package/dist/src/redteam/plugins/intent.js +44 -22
- package/dist/src/redteam/plugins/intent.js.map +1 -1
- package/dist/src/redteam/providers/crescendo/index.d.ts.map +1 -1
- package/dist/src/redteam/providers/crescendo/index.js +7 -2
- package/dist/src/redteam/providers/crescendo/index.js.map +1 -1
- package/dist/src/redteam/providers/goat.js +1 -1
- package/dist/src/redteam/providers/goat.js.map +1 -1
- package/dist/src/redteam/providers/iterative.d.ts.map +1 -1
- package/dist/src/redteam/providers/iterative.js +1 -1
- package/dist/src/redteam/providers/iterative.js.map +1 -1
- package/dist/src/redteam/providers/iterativeTree.js +1 -1
- package/dist/src/redteam/providers/iterativeTree.js.map +1 -1
- package/dist/src/server/routes/user.js +1 -1
- package/dist/src/server/routes/user.js.map +1 -1
- package/dist/src/server/server.js +1 -1
- package/dist/src/server/server.js.map +1 -1
- package/dist/src/telemetry.d.ts +12 -6
- package/dist/src/telemetry.d.ts.map +1 -1
- package/dist/src/telemetry.js +75 -78
- package/dist/src/telemetry.js.map +1 -1
- package/dist/src/util/config/load.js +4 -4
- package/dist/src/util/config/load.js.map +1 -1
- package/dist/src/util/testCaseReader.js +9 -9
- package/dist/src/util/testCaseReader.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +28 -30
|
@@ -36,19 +36,18 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
36
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
exports.DEFAULT_TURN_COUNT = exports.ArgsSchema = void 0;
|
|
39
|
+
exports.DEFAULT_TURN_COUNT = exports.ArgsSchema = exports.TargetPurposeDiscoveryResponseSchema = exports.DiscoveredPurposeSchema = exports.TargetPurposeDiscoveryRequestSchema = exports.TargetPurposeDiscoveryStateSchema = void 0;
|
|
40
40
|
exports.doTargetPurposeDiscovery = doTargetPurposeDiscovery;
|
|
41
41
|
exports.mergePurposes = mergePurposes;
|
|
42
42
|
exports.discoverCommand = discoverCommand;
|
|
43
43
|
const chalk_1 = __importDefault(require("chalk"));
|
|
44
44
|
const cli_progress_1 = __importDefault(require("cli-progress"));
|
|
45
|
-
const
|
|
45
|
+
const crypto_1 = require("crypto");
|
|
46
46
|
const dedent_1 = __importDefault(require("dedent"));
|
|
47
47
|
const fs = __importStar(require("fs"));
|
|
48
|
-
const js_yaml_1 = __importDefault(require("js-yaml"));
|
|
49
|
-
const path_1 = __importDefault(require("path"));
|
|
50
48
|
const zod_1 = require("zod");
|
|
51
49
|
const constants_1 = require("../../constants");
|
|
50
|
+
const evaluatorHelpers_1 = require("../../evaluatorHelpers");
|
|
52
51
|
const fetch_1 = require("../../fetch");
|
|
53
52
|
const accounts_1 = require("../../globalConfig/accounts");
|
|
54
53
|
const cloud_1 = require("../../globalConfig/cloud");
|
|
@@ -57,44 +56,40 @@ const providers_1 = require("../../providers");
|
|
|
57
56
|
const telemetry_1 = __importDefault(require("../../telemetry"));
|
|
58
57
|
const cloud_2 = require("../../util/cloud");
|
|
59
58
|
const load_1 = require("../../util/config/load");
|
|
60
|
-
const manage_1 = require("../../util/config/manage");
|
|
61
59
|
const invariant_1 = __importDefault(require("../../util/invariant"));
|
|
62
|
-
const constants_2 = require("../constants");
|
|
63
60
|
const remoteGeneration_1 = require("../remoteGeneration");
|
|
64
61
|
const remoteGeneration_2 = require("../remoteGeneration");
|
|
62
|
+
exports.TargetPurposeDiscoveryStateSchema = zod_1.z.object({
|
|
63
|
+
currentQuestionIndex: zod_1.z.number(),
|
|
64
|
+
answers: zod_1.z.array(zod_1.z.string()),
|
|
65
|
+
});
|
|
66
|
+
exports.TargetPurposeDiscoveryRequestSchema = zod_1.z.object({
|
|
67
|
+
state: exports.TargetPurposeDiscoveryStateSchema,
|
|
68
|
+
task: zod_1.z.literal('target-purpose-discovery'),
|
|
69
|
+
version: zod_1.z.string(),
|
|
70
|
+
email: zod_1.z.string().optional().nullable(),
|
|
71
|
+
});
|
|
72
|
+
exports.DiscoveredPurposeSchema = zod_1.z.object({
|
|
73
|
+
purpose: zod_1.z.string().nullable(),
|
|
74
|
+
limitations: zod_1.z.string().nullable(),
|
|
75
|
+
user: zod_1.z.string().nullable(),
|
|
76
|
+
tools: zod_1.z.array(zod_1.z.record(zod_1.z.any())).nullable(),
|
|
77
|
+
});
|
|
78
|
+
exports.TargetPurposeDiscoveryResponseSchema = zod_1.z.object({
|
|
79
|
+
done: zod_1.z.boolean(),
|
|
80
|
+
question: zod_1.z.string().optional(),
|
|
81
|
+
purpose: exports.DiscoveredPurposeSchema.optional(),
|
|
82
|
+
state: exports.TargetPurposeDiscoveryStateSchema,
|
|
83
|
+
});
|
|
65
84
|
exports.ArgsSchema = zod_1.z
|
|
66
85
|
.object({
|
|
67
86
|
config: zod_1.z.string().optional(),
|
|
68
|
-
output: zod_1.z.string().optional(),
|
|
69
87
|
target: zod_1.z.string().optional(),
|
|
70
|
-
preview: zod_1.z.boolean(),
|
|
71
|
-
turns: zod_1.z.number().optional(),
|
|
72
|
-
overwrite: zod_1.z.boolean(),
|
|
73
88
|
})
|
|
74
89
|
// Config and target are mutually exclusive:
|
|
75
90
|
.refine((data) => !(data.config && data.target), {
|
|
76
91
|
message: 'Cannot specify both config and target!',
|
|
77
92
|
path: ['config', 'target'],
|
|
78
|
-
})
|
|
79
|
-
// Either config or target must be provided:
|
|
80
|
-
.refine((data) => data.config || data.target, {
|
|
81
|
-
message: 'Either config or target must be provided!',
|
|
82
|
-
path: ['config', 'target'],
|
|
83
|
-
})
|
|
84
|
-
// `output` and `preview` are mutually exclusive:
|
|
85
|
-
.refine((data) => !(data.output && data.preview), {
|
|
86
|
-
message: 'Cannot specify both output and preview!',
|
|
87
|
-
path: ['output', 'preview'],
|
|
88
|
-
})
|
|
89
|
-
// `overwrite` can only be used if `output` is provided:
|
|
90
|
-
.refine((data) => !(data.overwrite && !data.output), {
|
|
91
|
-
message: 'Cannot specify overwrite without output!',
|
|
92
|
-
path: ['overwrite', 'output'],
|
|
93
|
-
})
|
|
94
|
-
// if `preview` is false, `output` must be provided:
|
|
95
|
-
.refine((data) => !(data.preview === false && !data.output), {
|
|
96
|
-
message: 'If preview is false, output must be provided!',
|
|
97
|
-
path: ['preview', 'output'],
|
|
98
93
|
});
|
|
99
94
|
// A larger turn count is more accurate (b/c more probes) but slower.
|
|
100
95
|
// TODO: Optimize this default to balance quality/runtime using the Discover eval.
|
|
@@ -108,108 +103,111 @@ exports.DEFAULT_TURN_COUNT = 5;
|
|
|
108
103
|
* @param maxTurns - The maximum number of turns to run the discovery process.
|
|
109
104
|
* @returns The purpose of the target.
|
|
110
105
|
*/
|
|
111
|
-
async function doTargetPurposeDiscovery(target,
|
|
112
|
-
const conversationHistory = [];
|
|
113
|
-
let turnCounter = 0;
|
|
106
|
+
async function doTargetPurposeDiscovery(target, prompt) {
|
|
114
107
|
const pbar = new cli_progress_1.default.SingleBar({
|
|
115
|
-
format: `
|
|
108
|
+
format: `Discovery phase - probing the target {bar} {percentage}% | {value}${exports.DEFAULT_TURN_COUNT ? '/{total}' : ''} turns`,
|
|
116
109
|
barCompleteChar: '\u2588',
|
|
117
110
|
barIncompleteChar: '\u2591',
|
|
118
111
|
hideCursor: true,
|
|
119
112
|
});
|
|
120
|
-
pbar.start(
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
113
|
+
pbar.start(exports.DEFAULT_TURN_COUNT, 0);
|
|
114
|
+
let done = false;
|
|
115
|
+
let question = undefined;
|
|
116
|
+
let purpose = undefined;
|
|
117
|
+
let state = exports.TargetPurposeDiscoveryStateSchema.parse({
|
|
118
|
+
currentQuestionIndex: 0,
|
|
119
|
+
answers: [],
|
|
120
|
+
});
|
|
121
|
+
let turn = 0;
|
|
122
|
+
while (!done && turn < 10) {
|
|
123
|
+
try {
|
|
124
|
+
turn++;
|
|
125
|
+
logger_1.default.debug(`[TargetPurposeDiscovery] Starting the purpose discovery loop, turn: ${turn}`);
|
|
126
|
+
const request = exports.TargetPurposeDiscoveryRequestSchema.parse({
|
|
127
|
+
state: {
|
|
128
|
+
currentQuestionIndex: state.currentQuestionIndex,
|
|
129
|
+
answers: state.answers,
|
|
130
|
+
},
|
|
131
|
+
task: 'target-purpose-discovery',
|
|
132
|
+
version: constants_1.VERSION,
|
|
133
|
+
email: (0, accounts_1.getUserEmail)(),
|
|
134
|
+
});
|
|
135
|
+
const response = await (0, fetch_1.fetchWithProxy)((0, remoteGeneration_1.getRemoteGenerationUrl)(), {
|
|
132
136
|
method: 'POST',
|
|
137
|
+
headers: {
|
|
138
|
+
'Content-Type': 'application/json',
|
|
139
|
+
Authorization: `Bearer ${cloud_1.cloudConfig.getApiKey()}`,
|
|
140
|
+
},
|
|
141
|
+
body: JSON.stringify(request),
|
|
133
142
|
});
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
logger_1.default.info(`\nPurpose:\n\n${chalk_1.default.green(purpose)}\n`);
|
|
139
|
-
return purpose;
|
|
143
|
+
if (!response.ok) {
|
|
144
|
+
const error = await response.text();
|
|
145
|
+
logger_1.default.error(`[TargetPurposeDiscovery] Error getting the next question from remote server: ${error}`);
|
|
146
|
+
continue;
|
|
140
147
|
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
148
|
+
const responseData = await response.json();
|
|
149
|
+
const response_ = exports.TargetPurposeDiscoveryResponseSchema.parse(responseData);
|
|
150
|
+
logger_1.default.debug(`[TargetPurposeDiscovery] Received response from remote server: ${JSON.stringify(response_, null, 2)}`);
|
|
151
|
+
done = response_.done;
|
|
152
|
+
question = response_.question;
|
|
153
|
+
purpose = response_.purpose;
|
|
154
|
+
state = response_.state;
|
|
155
|
+
if (!done) {
|
|
156
|
+
(0, invariant_1.default)(question, 'If its not done, then a quesation should always be defined, something is terribely wrong.');
|
|
157
|
+
const renderedPrompt = prompt
|
|
158
|
+
? await (0, evaluatorHelpers_1.renderPrompt)(prompt, { prompt: question }, {}, target)
|
|
159
|
+
: question;
|
|
160
|
+
const targetResponse = await target.callApi(renderedPrompt, {
|
|
161
|
+
prompt: { raw: question, label: 'Target Purpose Discovery Question' },
|
|
162
|
+
vars: { sessionId: (0, crypto_1.randomUUID)() },
|
|
163
|
+
});
|
|
164
|
+
if (targetResponse.error) {
|
|
165
|
+
logger_1.default.error(`[TargetPurposeDiscovery] Error from target: ${targetResponse.error}`);
|
|
166
|
+
if (turn > 10) {
|
|
167
|
+
logger_1.default.error('[TargetPurposeDiscovery] Too many retries, giving up.');
|
|
168
|
+
return undefined;
|
|
169
|
+
}
|
|
170
|
+
continue;
|
|
144
171
|
}
|
|
145
|
-
|
|
172
|
+
logger_1.default.debug(`[TargetPurposeDiscovery] Received response from target: ${JSON.stringify(targetResponse, null, 2)}`);
|
|
173
|
+
state.answers.push(targetResponse.output);
|
|
146
174
|
}
|
|
147
|
-
// Call the target with the question:
|
|
148
|
-
const response = await target.callApi(question);
|
|
149
|
-
logger_1.default.debug(JSON.stringify({ question, output: response.output }, null, 2));
|
|
150
|
-
conversationHistory.push({ type: 'target', content: response.output });
|
|
151
|
-
pbar.increment();
|
|
152
175
|
}
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
const res = await (0, fetch_1.fetchWithProxy)((0, remoteGeneration_1.getRemoteGenerationUrl)(), {
|
|
156
|
-
body: JSON.stringify({
|
|
157
|
-
task: 'target-purpose-discovery',
|
|
158
|
-
conversationHistory,
|
|
159
|
-
maxTurns,
|
|
160
|
-
version: constants_1.VERSION,
|
|
161
|
-
email: (0, accounts_1.getUserEmail)(),
|
|
162
|
-
forceReturn: true,
|
|
163
|
-
}),
|
|
164
|
-
headers: { 'Content-Type': 'application/json' },
|
|
165
|
-
method: 'POST',
|
|
166
|
-
});
|
|
167
|
-
const { purpose } = (await res.json());
|
|
168
|
-
if (purpose) {
|
|
169
|
-
return purpose;
|
|
176
|
+
catch (error) {
|
|
177
|
+
logger_1.default.error(`An unexpected error occurred during target discovery: ${error instanceof Error ? error.message : String(error)}\n${error instanceof Error ? error.stack : ''}`);
|
|
170
178
|
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
finally {
|
|
174
|
-
// Ensure progress bar is always stopped, even if an exception occurs
|
|
175
|
-
if (pbar) {
|
|
176
|
-
pbar.stop();
|
|
179
|
+
finally {
|
|
180
|
+
pbar.increment(1);
|
|
177
181
|
}
|
|
178
182
|
}
|
|
183
|
+
pbar.stop();
|
|
184
|
+
return purpose;
|
|
179
185
|
}
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
(0, invariant_1.default)(cloud_1.cloudConfig.isEnabled(), 'Cloud config should have been enabled for a target to be provided');
|
|
189
|
-
const url = `${cloud_1.cloudConfig.getApiHost()}/api/v1/providers/${targetId}`;
|
|
190
|
-
logger_1.default.debug(`Saving purpose to ${url}`);
|
|
191
|
-
const res = await (0, fetch_1.fetchWithProxy)(url, {
|
|
192
|
-
method: 'PATCH',
|
|
193
|
-
headers: {
|
|
194
|
-
'Content-Type': 'application/json',
|
|
195
|
-
Authorization: `Bearer ${cloud_1.cloudConfig.getApiKey()}`,
|
|
196
|
-
},
|
|
197
|
-
body: JSON.stringify({ applicationDescription: { purpose } }),
|
|
198
|
-
});
|
|
199
|
-
if (res.ok) {
|
|
200
|
-
logger_1.default.info('Purpose updated');
|
|
186
|
+
function mergePurposes(humanDefinedPurpose, discoveredPurpose) {
|
|
187
|
+
let purpose = '';
|
|
188
|
+
if (humanDefinedPurpose) {
|
|
189
|
+
purpose += `This purpose was defined by the user and should be trusted and treated as absolute truth:
|
|
190
|
+
<HumanDefinedPurpose>
|
|
191
|
+
${humanDefinedPurpose}
|
|
192
|
+
</HumanDefinedPurpose>
|
|
193
|
+
`;
|
|
201
194
|
}
|
|
202
|
-
|
|
203
|
-
|
|
195
|
+
if (discoveredPurpose) {
|
|
196
|
+
purpose += `This purpose was discovered by the agent from conversations with the target. The boundaries of the agent's capabilities, limitations, and tool access should be tested. If there are any discrepancies, the user-defined purpose should be trusted and treated as absolute truth:
|
|
197
|
+
<AgentDiscoveredPurpose>
|
|
198
|
+
${discoveredPurpose.purpose ? `The target believes its purpose is: ${discoveredPurpose.purpose}` : ''}
|
|
199
|
+
${discoveredPurpose.limitations ? `The target believes its limitations are: ${discoveredPurpose.limitations}` : ''}
|
|
200
|
+
${discoveredPurpose.tools ? `The target divulged access to these tools: ${JSON.stringify(discoveredPurpose.tools, null, 2)}` : ''}
|
|
201
|
+
${discoveredPurpose.user ? `The target believes the user of the application is: ${discoveredPurpose.user}` : ''}
|
|
202
|
+
</AgentDiscoveredPurpose>
|
|
203
|
+
`;
|
|
204
204
|
}
|
|
205
|
-
|
|
206
|
-
function mergePurposes(humanDefinedPurpose, discoveredPurpose) {
|
|
207
|
-
return `${humanDefinedPurpose}\n\nDiscovered Purpose:\n\n${discoveredPurpose}`;
|
|
205
|
+
return purpose;
|
|
208
206
|
}
|
|
209
207
|
/**
|
|
210
208
|
* Registers the `discover` command with the CLI.
|
|
211
209
|
*/
|
|
212
|
-
function discoverCommand(program) {
|
|
210
|
+
function discoverCommand(program, defaultConfig, defaultConfigPath) {
|
|
213
211
|
program
|
|
214
212
|
.command('discover')
|
|
215
213
|
.description((0, dedent_1.default) `
|
|
@@ -219,21 +217,8 @@ function discoverCommand(program) {
|
|
|
219
217
|
and the target will be discovered from the first provider in that config.
|
|
220
218
|
`)
|
|
221
219
|
.option('-c, --config <path>', 'Path to `promptfooconfig.yaml` configuration file.')
|
|
222
|
-
.option('-o, --output <path>', 'Path to output file. Discovered purpose will be appended to the file if it already exists.', constants_2.DEFAULT_OUTPUT_PATH)
|
|
223
|
-
.option('--overwrite', 'Overwrite the existing purpose if it already exists.', false)
|
|
224
220
|
.option('-t, --target <id>', 'UUID of a Cloud-defined target to run the discovery on')
|
|
225
|
-
.option('--preview', 'Preview discovery results without writing to an output file', false)
|
|
226
|
-
.addOption(new commander_1.Option('--turns <turns>', 'A maximum number of turns to run the discovery process. Lower is faster but less accurate.')
|
|
227
|
-
.argParser(Number.parseInt)
|
|
228
|
-
.default(exports.DEFAULT_TURN_COUNT))
|
|
229
221
|
.action(async (rawArgs) => {
|
|
230
|
-
// If preview is true and output is DEFAULT_OUTPUT_PATH, set output to undefined to satisfy
|
|
231
|
-
// the schema. Defaults are defined within the `option` definitions to include them within the
|
|
232
|
-
// help message; however the schema enforces combinations of options that are mutually exclusive,
|
|
233
|
-
// such as `output` and `preview`.
|
|
234
|
-
if (rawArgs.preview && rawArgs.output === constants_2.DEFAULT_OUTPUT_PATH) {
|
|
235
|
-
rawArgs.output = undefined;
|
|
236
|
-
}
|
|
237
222
|
// Check that remote generation is enabled:
|
|
238
223
|
if ((0, remoteGeneration_2.neverGenerateRemote)()) {
|
|
239
224
|
logger_1.default.error((0, dedent_1.default) `
|
|
@@ -262,7 +247,6 @@ function discoverCommand(program) {
|
|
|
262
247
|
// a single target at a time.
|
|
263
248
|
let target = undefined;
|
|
264
249
|
// Fallback to the default config path:
|
|
265
|
-
const fallbackConfigPath = path_1.default.join(process.cwd(), 'promptfooconfig.yaml');
|
|
266
250
|
// If user provides a config, read the target from it:
|
|
267
251
|
if (args.config) {
|
|
268
252
|
// Validate that the config is a valid path:
|
|
@@ -274,7 +258,7 @@ function discoverCommand(program) {
|
|
|
274
258
|
throw new Error(`Config is invalid at ${args.config}`);
|
|
275
259
|
}
|
|
276
260
|
if (!config.providers) {
|
|
277
|
-
throw new Error('Config must contain
|
|
261
|
+
throw new Error('Config must contain a target');
|
|
278
262
|
}
|
|
279
263
|
const providers = await (0, providers_1.loadApiProviders)(config.providers);
|
|
280
264
|
target = providers[0];
|
|
@@ -286,75 +270,51 @@ function discoverCommand(program) {
|
|
|
286
270
|
target = await (0, providers_1.loadApiProvider)(providerOptions.id, { options: providerOptions });
|
|
287
271
|
}
|
|
288
272
|
// Check the current working directory for a promptfooconfig.yaml file:
|
|
289
|
-
else if (
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
throw new Error(`Config is invalid at ${fallbackConfigPath}`);
|
|
273
|
+
else if (defaultConfig) {
|
|
274
|
+
if (!defaultConfig) {
|
|
275
|
+
throw new Error(`Config is invalid at ${defaultConfigPath}`);
|
|
293
276
|
}
|
|
294
|
-
if (!
|
|
295
|
-
throw new Error('Config must contain
|
|
277
|
+
if (!defaultConfig.providers) {
|
|
278
|
+
throw new Error('Config must contain a target or provider');
|
|
296
279
|
}
|
|
297
|
-
const providers = await (0, providers_1.loadApiProviders)(
|
|
280
|
+
const providers = await (0, providers_1.loadApiProviders)(defaultConfig.providers);
|
|
298
281
|
target = providers[0];
|
|
299
282
|
// Alert the user that we're using a config from the current working directory:
|
|
300
|
-
logger_1.default.info(`Using config from ${chalk_1.default.italic(
|
|
283
|
+
logger_1.default.info(`Using config from ${chalk_1.default.italic(defaultConfigPath)}`);
|
|
284
|
+
}
|
|
285
|
+
else {
|
|
286
|
+
logger_1.default.error('No config found, please specify a config file with the --config flag, a target with the --target flag, or run this command from a directory with a promptfooconfig.yaml file.');
|
|
287
|
+
process.exitCode = 1;
|
|
288
|
+
return;
|
|
301
289
|
}
|
|
302
|
-
// At this point, we should have at least one target:
|
|
303
|
-
(0, invariant_1.default)(target != undefined, 'An error occurred loading the target config');
|
|
304
290
|
// Discover the purpose for the target:
|
|
305
291
|
let purpose = undefined;
|
|
306
292
|
try {
|
|
307
|
-
purpose = await doTargetPurposeDiscovery(target
|
|
293
|
+
purpose = await doTargetPurposeDiscovery(target);
|
|
308
294
|
}
|
|
309
295
|
catch (error) {
|
|
310
296
|
logger_1.default.error(`An unexpected error occurred during target discovery: ${error instanceof Error ? error.message : String(error)}\n${error instanceof Error ? error.stack : ''}`);
|
|
311
297
|
process.exit(1);
|
|
312
298
|
}
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
await saveCloudTargetPurpose(args.target, purpose);
|
|
299
|
+
if (purpose) {
|
|
300
|
+
if (purpose.purpose) {
|
|
301
|
+
logger_1.default.info(chalk_1.default.bold(chalk_1.default.green('The target believes its purpose is:')));
|
|
302
|
+
logger_1.default.info(chalk_1.default.bold(purpose.purpose + '\n\n'));
|
|
318
303
|
}
|
|
319
|
-
|
|
320
|
-
(
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
logger_1.default.
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
if (args.overwrite) {
|
|
331
|
-
logger_1.default.warn((0, dedent_1.default) `
|
|
332
|
-
Output file already contains a value at \`redteam.purpose\`; overwriting it.
|
|
333
|
-
`);
|
|
334
|
-
}
|
|
335
|
-
else {
|
|
336
|
-
logger_1.default.warn((0, dedent_1.default) `
|
|
337
|
-
Output file already contains a value at \`redteam.purpose\`; appending discovered purpose to it.
|
|
338
|
-
|
|
339
|
-
To overwrite the existing purpose, use the \`--overwrite\` flag.
|
|
340
|
-
`);
|
|
341
|
-
}
|
|
342
|
-
}
|
|
343
|
-
existingYaml['redteam'] = {
|
|
344
|
-
...(existingYaml['redteam'] || {}),
|
|
345
|
-
purpose: existingPurpose && !args.overwrite
|
|
346
|
-
? mergePurposes(existingPurpose, purpose)
|
|
347
|
-
: purpose,
|
|
348
|
-
};
|
|
349
|
-
(0, manage_1.writePromptfooConfig)(existingYaml, args.output);
|
|
350
|
-
}
|
|
351
|
-
else {
|
|
352
|
-
// Create a new config file with the purpose.
|
|
353
|
-
(0, manage_1.writePromptfooConfig)({ redteam: { purpose } }, args.output);
|
|
354
|
-
}
|
|
355
|
-
logger_1.default.info(`\nPurpose written to ${chalk_1.default.italic(args.output)}`);
|
|
304
|
+
if (purpose.limitations) {
|
|
305
|
+
logger_1.default.info(chalk_1.default.bold(chalk_1.default.green('The target believes its limitations to be:')));
|
|
306
|
+
logger_1.default.info(purpose.limitations + '\n\n');
|
|
307
|
+
}
|
|
308
|
+
if (purpose.tools) {
|
|
309
|
+
logger_1.default.info(chalk_1.default.bold(chalk_1.default.green('The target divulged access to these tools:')));
|
|
310
|
+
logger_1.default.info(JSON.stringify(purpose.tools, null, 2));
|
|
311
|
+
}
|
|
312
|
+
if (purpose.user) {
|
|
313
|
+
logger_1.default.info(chalk_1.default.bold(chalk_1.default.green('The target believes the user of the application is:')));
|
|
314
|
+
logger_1.default.info(purpose.user + '\n\n');
|
|
356
315
|
}
|
|
357
316
|
}
|
|
317
|
+
process.exit();
|
|
358
318
|
});
|
|
359
319
|
}
|
|
360
320
|
//# sourceMappingURL=discover.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"discover.js","sourceRoot":"","sources":["../../../../src/redteam/commands/discover.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"discover.js","sourceRoot":"","sources":["../../../../src/redteam/commands/discover.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2EA,4DA8GC;AAED,sCAuBC;AAKD,0CAwIC;AA/VD,kDAA0B;AAC1B,gEAAuC;AAEvC,mCAAoC;AACpC,oDAA4B;AAC5B,uCAAyB;AACzB,6BAAwB;AACxB,+CAA0C;AAC1C,6DAAsD;AACtD,uCAA6C;AAC7C,0DAA2D;AAC3D,oDAAuD;AACvD,0DAAkC;AAClC,+CAAoE;AACpE,gEAAwC;AAExC,4CAAwD;AACxD,iDAAoD;AACpD,qEAA6C;AAC7C,0DAA6D;AAC7D,0DAA0D;AAE7C,QAAA,iCAAiC,GAAG,OAAC,CAAC,MAAM,CAAC;IACxD,oBAAoB,EAAE,OAAC,CAAC,MAAM,EAAE;IAChC,OAAO,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;CAC7B,CAAC,CAAC;AAEU,QAAA,mCAAmC,GAAG,OAAC,CAAC,MAAM,CAAC;IAC1D,KAAK,EAAE,yCAAiC;IACxC,IAAI,EAAE,OAAC,CAAC,OAAO,CAAC,0BAA0B,CAAC;IAC3C,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE;IACnB,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACxC,CAAC,CAAC;AACU,QAAA,uBAAuB,GAAG,OAAC,CAAC,MAAM,CAAC;IAC9C,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,KAAK,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;CAC7C,CAAC,CAAC;AAEU,QAAA,oCAAoC,GAAG,OAAC,CAAC,MAAM,CAAC;IAC3D,IAAI,EAAE,OAAC,CAAC,OAAO,EAAE;IACjB,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,+BAAuB,CAAC,QAAQ,EAAE;IAC3C,KAAK,EAAE,yCAAiC;CACzC,CAAC,CAAC;AAIU,QAAA,UAAU,GAAG,OAAC;KACxB,MAAM,CAAC;IACN,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC9B,CAAC;IACF,4CAA4C;KAC3C,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE;IAC/C,OAAO,EAAE,wCAAwC;IACjD,IAAI,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;CAC3B,CAAC,CAAC;AAIL,qEAAqE;AACrE,kFAAkF;AAClF,4DAA4D;AAC/C,QAAA,kBAAkB,GAAG,CAAC,CAAC;AAEpC;;;;;;;GAOG;AACI,KAAK,UAAU,wBAAwB,CAC5C,MAAmB,EACnB,MAAe;IAEf,MAAM,IAAI,GAAG,IAAI,sBAAW,CAAC,SAAS,CAAC;QACrC,MAAM,EAAE,qEAAqE,0BAAkB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,QAAQ;QACzH,eAAe,EAAE,QAAQ;QACzB,iBAAiB,EAAE,QAAQ;QAC3B,UAAU,EAAE,IAAI;KACjB,CAAC,CAAC;IAEH,IAAI,CAAC,KAAK,CAAC,0BAAkB,EAAE,CAAC,CAAC,CAAC;IAElC,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,QAAQ,GAAuB,SAAS,CAAC;IAC7C,IAAI,OAAO,GAAkC,SAAS,CAAC;IACvD,IAAI,KAAK,GAAG,yCAAiC,CAAC,KAAK,CAAC;QAClD,oBAAoB,EAAE,CAAC;QACvB,OAAO,EAAE,EAAE;KACZ,CAAC,CAAC;IACH,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,OAAO,CAAC,IAAI,IAAI,IAAI,GAAG,EAAE,EAAE,CAAC;QAC1B,IAAI,CAAC;YACH,IAAI,EAAE,CAAC;YACP,gBAAM,CAAC,KAAK,CAAC,uEAAuE,IAAI,EAAE,CAAC,CAAC;YAC5F,MAAM,OAAO,GAAG,2CAAmC,CAAC,KAAK,CAAC;gBACxD,KAAK,EAAE;oBACL,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;oBAChD,OAAO,EAAE,KAAK,CAAC,OAAO;iBACvB;gBACD,IAAI,EAAE,0BAA0B;gBAChC,OAAO,EAAE,mBAAO;gBAChB,KAAK,EAAE,IAAA,uBAAY,GAAE;aACtB,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,MAAM,IAAA,sBAAc,EAAC,IAAA,yCAAsB,GAAE,EAAE;gBAC9D,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,mBAAW,CAAC,SAAS,EAAE,EAAE;iBACnD;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;aAC9B,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,gBAAM,CAAC,KAAK,CACV,gFAAgF,KAAK,EAAE,CACxF,CAAC;gBACF,SAAS;YACX,CAAC;YAED,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC3C,MAAM,SAAS,GAAG,4CAAoC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YAE3E,gBAAM,CAAC,KAAK,CACV,kEAAkE,IAAI,CAAC,SAAS,CAC9E,SAAS,EACT,IAAI,EACJ,CAAC,CACF,EAAE,CACJ,CAAC;YACF,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC;YACtB,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC;YAC9B,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC;YAC5B,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;YAExB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,IAAA,mBAAS,EACP,QAAQ,EACR,2FAA2F,CAC5F,CAAC;gBACF,MAAM,cAAc,GAAG,MAAM;oBAC3B,CAAC,CAAC,MAAM,IAAA,+BAAY,EAAC,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC;oBAC9D,CAAC,CAAC,QAAQ,CAAC;gBACb,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE;oBAC1D,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,mCAAmC,EAAE;oBACrE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAA,mBAAU,GAAE,EAAE;iBAClC,CAAC,CAAC;gBACH,IAAI,cAAc,CAAC,KAAK,EAAE,CAAC;oBACzB,gBAAM,CAAC,KAAK,CAAC,+CAA+C,cAAc,CAAC,KAAK,EAAE,CAAC,CAAC;oBACpF,IAAI,IAAI,GAAG,EAAE,EAAE,CAAC;wBACd,gBAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;wBACtE,OAAO,SAAS,CAAC;oBACnB,CAAC;oBACD,SAAS;gBACX,CAAC;gBACD,gBAAM,CAAC,KAAK,CACV,2DAA2D,IAAI,CAAC,SAAS,CACvE,cAAc,EACd,IAAI,EACJ,CAAC,CACF,EAAE,CACJ,CAAC;gBACF,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,gBAAM,CAAC,KAAK,CACV,yDAAyD,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAC7G,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EACzC,EAAE,CACH,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IACD,IAAI,CAAC,IAAI,EAAE,CAAC;IAEZ,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAgB,aAAa,CAC3B,mBAAuC,EACvC,iBAAkD;IAElD,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,mBAAmB,EAAE,CAAC;QACxB,OAAO,IAAI;;MAET,mBAAmB;;KAEpB,CAAC;IACJ,CAAC;IACD,IAAI,iBAAiB,EAAE,CAAC;QACtB,OAAO,IAAI;;MAET,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,uCAAuC,iBAAiB,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE;MACnG,iBAAiB,CAAC,WAAW,CAAC,CAAC,CAAC,4CAA4C,iBAAiB,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE;MAChH,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,8CAA8C,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;MAC/H,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,uDAAuD,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE;;KAE9G,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,SAAgB,eAAe,CAC7B,OAAgB,EAChB,aAAqC,EACrC,iBAAqC;IAErC,OAAO;SACJ,OAAO,CAAC,UAAU,CAAC;SACnB,WAAW,CACV,IAAA,gBAAM,EAAA;;;;;OAKL,CACF;SACA,MAAM,CAAC,qBAAqB,EAAE,oDAAoD,CAAC;SACnF,MAAM,CAAC,mBAAmB,EAAE,wDAAwD,CAAC;SACrF,MAAM,CAAC,KAAK,EAAE,OAAa,EAAE,EAAE;QAC9B,2CAA2C;QAC3C,IAAI,IAAA,sCAAmB,GAAE,EAAE,CAAC;YAC1B,gBAAM,CAAC,KAAK,CAAC,IAAA,gBAAM,EAAA;;;;SAIlB,CAAC,CAAC;YACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,0BAA0B;QAC1B,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,kBAAU,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACrE,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,gBAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YACjC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC7B,gBAAM,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;YACH,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;QAED,oBAAoB;QACpB,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,kBAAkB;SACzB,CAAC,CAAC;QAEH,IAAI,MAAM,GAAyB,IAAI,CAAC;QACxC,6FAA6F;QAC7F,6BAA6B;QAC7B,IAAI,MAAM,GAA4B,SAAS,CAAC;QAChD,uCAAuC;QAEvC,sDAAsD;QACtD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,4CAA4C;YAC5C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACxD,CAAC;YAED,MAAM,GAAG,MAAM,IAAA,iBAAU,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEvC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;YAClD,CAAC;YAED,MAAM,SAAS,GAAG,MAAM,IAAA,4BAAgB,EAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAE3D,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC;QACD,sDAAsD;aACjD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACrB,6CAA6C;YAC7C,MAAM,eAAe,GAAG,MAAM,IAAA,4BAAoB,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAChE,MAAM,GAAG,MAAM,IAAA,2BAAe,EAAC,eAAe,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;QACnF,CAAC;QACD,uEAAuE;aAClE,IAAI,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC,aAAa,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,wBAAwB,iBAAiB,EAAE,CAAC,CAAC;YAC/D,CAAC;YAED,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;YAC9D,CAAC;YAED,MAAM,SAAS,GAAG,MAAM,IAAA,4BAAgB,EAAC,aAAa,CAAC,SAAS,CAAC,CAAC;YAClE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YAEtB,+EAA+E;YAC/E,gBAAM,CAAC,IAAI,CAAC,qBAAqB,eAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;QACtE,CAAC;aAAM,CAAC;YACN,gBAAM,CAAC,KAAK,CACV,+KAA+K,CAChL,CAAC;YACF,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;QAED,uCAAuC;QACvC,IAAI,OAAO,GAAoC,SAAS,CAAC;QACzD,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,wBAAwB,CAAC,MAAM,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,gBAAM,CAAC,KAAK,CACV,yDAAyD,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAC7G,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EACzC,EAAE,CACH,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;gBACpB,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,IAAI,CAAC,eAAK,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC,CAAC,CAAC;gBAC5E,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC;YACpD,CAAC;YACD,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;gBACxB,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,IAAI,CAAC,eAAK,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC,CAAC,CAAC;gBACnF,gBAAM,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,MAAM,CAAC,CAAC;YAC5C,CAAC;YACD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBAClB,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,IAAI,CAAC,eAAK,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC,CAAC,CAAC;gBACnF,gBAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACtD,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,IAAI,CAAC,eAAK,CAAC,KAAK,CAAC,qDAAqD,CAAC,CAAC,CAC/E,CAAC;gBACF,gBAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,EAAE,CAAC;IACjB,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../../src/redteam/commands/generate.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAgBzC,OAAO,KAAK,EAA0B,aAAa,EAAE,MAAM,aAAa,CAAC;AAczE,OAAO,KAAK,EACV,yBAAyB,EAI1B,MAAM,UAAU,CAAC;AAQlB,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,OAAO,CAAC,yBAAyB,CAAC,GAC1C,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../../src/redteam/commands/generate.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAgBzC,OAAO,KAAK,EAA0B,aAAa,EAAE,MAAM,aAAa,CAAC;AAczE,OAAO,KAAK,EACV,yBAAyB,EAI1B,MAAM,UAAU,CAAC;AAQlB,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,OAAO,CAAC,yBAAyB,CAAC,GAC1C,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,CA6UxC;AAED,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,SAAS,GAAG,UAAU,EAC/B,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,EACrC,iBAAiB,EAAE,MAAM,GAAG,SAAS,QA0HtC"}
|
|
@@ -76,7 +76,6 @@ async function doGenerateRedteam(options) {
|
|
|
76
76
|
let redteamConfig;
|
|
77
77
|
const configPath = options.config || options.defaultConfigPath;
|
|
78
78
|
const outputPath = options.output || 'redteam.yaml';
|
|
79
|
-
let finalPurpose = options.purpose;
|
|
80
79
|
// Check for updates to the config file and decide whether to generate
|
|
81
80
|
let shouldGenerate = options.force;
|
|
82
81
|
if (!options.force && fs.existsSync(outputPath) && configPath && fs.existsSync(configPath)) {
|
|
@@ -95,15 +94,13 @@ async function doGenerateRedteam(options) {
|
|
|
95
94
|
else {
|
|
96
95
|
shouldGenerate = true;
|
|
97
96
|
}
|
|
97
|
+
let discoveredPurpose;
|
|
98
98
|
if (configPath) {
|
|
99
99
|
const resolved = await (0, load_1.resolveConfigs)({
|
|
100
100
|
config: [configPath],
|
|
101
101
|
}, options.defaultConfig || {});
|
|
102
102
|
testSuite = resolved.testSuite;
|
|
103
103
|
redteamConfig = resolved.config.redteam;
|
|
104
|
-
if (redteamConfig?.purpose) {
|
|
105
|
-
finalPurpose = redteamConfig.purpose;
|
|
106
|
-
}
|
|
107
104
|
// If automatic purpose discovery is enabled, remote generation is enabled, and a config is provided that contains at least one target,
|
|
108
105
|
// discover the purpose from the target:
|
|
109
106
|
if (!(0, envars_1.getEnvBool)('PROMPTFOO_DISABLE_REDTEAM_PURPOSE_DISCOVERY_AGENT', true) &&
|
|
@@ -113,14 +110,13 @@ async function doGenerateRedteam(options) {
|
|
|
113
110
|
(0, invariant_1.default)(resolved.config.providers.length > 0, 'At least one provider must be provided in the config file');
|
|
114
111
|
const providers = await (0, providers_1.loadApiProviders)(resolved.config.providers);
|
|
115
112
|
try {
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
: generatedPurpose;
|
|
113
|
+
if (testSuite.prompts.length > 1) {
|
|
114
|
+
logger_1.default.warn('More than one prompt provided, only the first prompt will be used for purpose discovery');
|
|
115
|
+
}
|
|
116
|
+
discoveredPurpose = await (0, discover_1.doTargetPurposeDiscovery)(providers[0], testSuite.prompts[0]);
|
|
121
117
|
}
|
|
122
118
|
catch (error) {
|
|
123
|
-
logger_1.default.error(`
|
|
119
|
+
logger_1.default.error(`Discovery failed from error, skipping: ${error instanceof Error ? error.message : String(error)}`);
|
|
124
120
|
}
|
|
125
121
|
}
|
|
126
122
|
}
|
|
@@ -136,6 +132,7 @@ async function doGenerateRedteam(options) {
|
|
|
136
132
|
logger_1.default.info(chalk_1.default.red(`\nCan't generate without configuration - run ${chalk_1.default.yellow.bold((0, util_1.isRunningUnderNpx)() ? 'npx promptfoo redteam init' : 'promptfoo redteam init')} first`));
|
|
137
133
|
return null;
|
|
138
134
|
}
|
|
135
|
+
const mergedPurpose = (0, discover_1.mergePurposes)(redteamConfig?.purpose ?? options.purpose, discoveredPurpose);
|
|
139
136
|
const startTime = Date.now();
|
|
140
137
|
telemetry_1.default.record('command_used', {
|
|
141
138
|
name: 'generate redteam - started',
|
|
@@ -144,6 +141,7 @@ async function doGenerateRedteam(options) {
|
|
|
144
141
|
plugins: redteamConfig?.plugins?.map((p) => (typeof p === 'string' ? p : p.id)) || [],
|
|
145
142
|
strategies: redteamConfig?.strategies?.map((s) => (typeof s === 'string' ? s : s.id)) || [],
|
|
146
143
|
});
|
|
144
|
+
await telemetry_1.default.send();
|
|
147
145
|
let plugins;
|
|
148
146
|
// If plugins are defined in the config file
|
|
149
147
|
if (redteamConfig?.plugins && redteamConfig.plugins.length > 0) {
|
|
@@ -201,7 +199,7 @@ async function doGenerateRedteam(options) {
|
|
|
201
199
|
entities: redteamConfig?.entities,
|
|
202
200
|
plugins,
|
|
203
201
|
provider: redteamConfig?.provider || options.provider,
|
|
204
|
-
purpose:
|
|
202
|
+
purpose: mergedPurpose,
|
|
205
203
|
strategies: strategyObjs,
|
|
206
204
|
delay: redteamConfig?.delay || options.delay,
|
|
207
205
|
sharing: redteamConfig?.sharing || options.sharing,
|
|
@@ -341,6 +339,7 @@ async function doGenerateRedteam(options) {
|
|
|
341
339
|
plugins: plugins.map((p) => p.id),
|
|
342
340
|
strategies: strategies.map((s) => (typeof s === 'string' ? s : s.id)),
|
|
343
341
|
});
|
|
342
|
+
await telemetry_1.default.send();
|
|
344
343
|
return ret;
|
|
345
344
|
}
|
|
346
345
|
function redteamGenerateCommand(program, command, defaultConfig, defaultConfigPath) {
|