promptfoo 0.72.0 → 0.72.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/dist/package.json +2 -2
- package/dist/src/commands/generate/redteam.d.ts +43 -13
- package/dist/src/commands/generate/redteam.d.ts.map +1 -1
- package/dist/src/commands/generate/redteam.js +63 -22
- package/dist/src/commands/generate/redteam.js.map +1 -1
- package/dist/src/commands/redteam.d.ts +0 -7
- package/dist/src/commands/redteam.d.ts.map +1 -1
- package/dist/src/commands/redteam.js +72 -89
- package/dist/src/commands/redteam.js.map +1 -1
- package/dist/src/commands/share.d.ts.map +1 -1
- package/dist/src/commands/share.js +4 -1
- package/dist/src/commands/share.js.map +1 -1
- package/dist/src/database.d.ts +1129 -3
- package/dist/src/database.d.ts.map +1 -1
- package/dist/src/onboarding.d.ts.map +1 -1
- package/dist/src/onboarding.js +23 -36
- package/dist/src/onboarding.js.map +1 -1
- package/dist/src/providers/bedrock.js +1 -1
- package/dist/src/providers/bedrock.js.map +1 -1
- package/dist/src/providers/manualInput.d.ts.map +1 -1
- package/dist/src/providers/manualInput.js +3 -22
- package/dist/src/providers/manualInput.js.map +1 -1
- package/dist/src/providers.d.ts.map +1 -1
- package/dist/src/providers.js +4 -5
- package/dist/src/providers.js.map +1 -1
- package/dist/src/redteam/constants.d.ts +2 -2
- package/dist/src/redteam/constants.d.ts.map +1 -1
- package/dist/src/redteam/constants.js +1 -1
- package/dist/src/redteam/constants.js.map +1 -1
- package/dist/src/redteam/index.d.ts +2 -6
- package/dist/src/redteam/index.d.ts.map +1 -1
- package/dist/src/redteam/index.js +7 -16
- package/dist/src/redteam/index.js.map +1 -1
- package/dist/src/redteam/methods/injections.d.ts.map +1 -1
- package/dist/src/redteam/methods/injections.js +10 -0
- package/dist/src/redteam/methods/injections.js.map +1 -1
- package/dist/src/types.d.ts +30597 -174
- package/dist/src/types.d.ts.map +1 -1
- package/dist/src/types.js +242 -1
- package/dist/src/types.js.map +1 -1
- package/dist/src/util/index.d.ts +520 -2
- package/dist/src/util/index.d.ts.map +1 -1
- package/dist/src/web/nextui/404/index.html +1 -1
- package/dist/src/web/nextui/404.html +1 -1
- package/dist/src/web/nextui/_next/static/chunks/737-538f50dc31cc8c49.js +1 -0
- package/dist/src/web/nextui/_next/static/chunks/app/auth/login/{page-2e6452987650f232.js → page-2e2c0c725127ea2d.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/app/auth/signup/{page-d81a6cfb22f8b8a2.js → page-ccdb1f6890601666.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/app/datasets/page-642db4f4a8e8ba40.js +1 -0
- package/dist/src/web/nextui/_next/static/chunks/app/eval/[id]/not-found-ce320e6d1e6d1d23.js +1 -0
- package/dist/src/web/nextui/_next/static/chunks/app/eval/[id]/{page-81aa116b34ef4a6b.js → page-0a39d3450aa09dda.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/app/eval/page-32eaa14d2384c5b0.js +1 -0
- package/dist/src/web/nextui/_next/static/chunks/app/{layout-ed6af4caf66026a5.js → layout-4282b1d33566e258.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/app/{page-122e9cfa52eb218a.js → page-5650318e57510b6c.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/app/progress/{page-aa447776a6487d50.js → page-462526776efd6fd6.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/app/prompts/{page-ac63288a243a9872.js → page-0ae78bc0bf7c56bc.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/app/report/{page-be70065fb1472831.js → page-e4f7851d41eb0f92.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/app/setup/{page-b19fc52e084a3efe.js → page-6cde750ec428cd75.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/{main-app-7a1376166cb8b72e.js → main-app-929a26b3c8cd3f7a.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/{webpack-8a9bc9ee0defb756.js → webpack-c9f728822666f852.js} +1 -1
- package/dist/src/web/nextui/_next/static/css/5bd2f45de1f3ba83.css +1 -0
- package/dist/src/web/nextui/auth/login/index.html +1 -1
- package/dist/src/web/nextui/auth/login/index.txt +6 -6
- package/dist/src/web/nextui/auth/signup/index.html +1 -1
- package/dist/src/web/nextui/auth/signup/index.txt +6 -6
- package/dist/src/web/nextui/datasets/index.html +1 -1
- package/dist/src/web/nextui/datasets/index.txt +6 -6
- package/dist/src/web/nextui/eval/index.html +1 -1
- package/dist/src/web/nextui/eval/index.txt +6 -6
- package/dist/src/web/nextui/index.html +1 -1
- package/dist/src/web/nextui/index.txt +5 -5
- package/dist/src/web/nextui/progress/index.html +1 -1
- package/dist/src/web/nextui/progress/index.txt +6 -6
- package/dist/src/web/nextui/prompts/index.html +1 -1
- package/dist/src/web/nextui/prompts/index.txt +6 -6
- package/dist/src/web/nextui/report/index.html +1 -1
- package/dist/src/web/nextui/report/index.txt +6 -6
- package/dist/src/web/nextui/setup/index.html +1 -1
- package/dist/src/web/nextui/setup/index.txt +7 -7
- package/package.json +2 -2
- package/dist/src/web/nextui/_next/static/chunks/737-dbd25d447fbc5006.js +0 -1
- package/dist/src/web/nextui/_next/static/chunks/app/datasets/page-73b93abb6a46ca0e.js +0 -1
- package/dist/src/web/nextui/_next/static/chunks/app/eval/[id]/not-found-fe10d5df88bc44ef.js +0 -1
- package/dist/src/web/nextui/_next/static/chunks/app/eval/page-56e122e6f2a64557.js +0 -1
- package/dist/src/web/nextui/_next/static/css/106779eb64615639.css +0 -1
- /package/dist/src/web/nextui/_next/static/{9oVNcJw_yPT1Zu44NMAgc → 8E4q0Tmu22G1wENZyXEq0}/_buildManifest.js +0 -0
- /package/dist/src/web/nextui/_next/static/{9oVNcJw_yPT1Zu44NMAgc → 8E4q0Tmu22G1wENZyXEq0}/_ssgManifest.js +0 -0
package/LICENSE
CHANGED
package/dist/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "promptfoo",
|
|
3
3
|
"description": "LLM eval & testing toolkit",
|
|
4
4
|
"author": "Ian Webster",
|
|
5
|
-
"version": "0.72.
|
|
5
|
+
"version": "0.72.1",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"type": "commonjs",
|
|
8
8
|
"repository": {
|
|
@@ -134,7 +134,7 @@
|
|
|
134
134
|
"fast-xml-parser": "^4.4.0",
|
|
135
135
|
"fastest-levenshtein": "^1.0.16",
|
|
136
136
|
"glob": "^10.4.3",
|
|
137
|
-
"inquirer": "^
|
|
137
|
+
"inquirer": "^10.0.2",
|
|
138
138
|
"js-yaml": "^4.1.0",
|
|
139
139
|
"mathjs": "^13.0.2",
|
|
140
140
|
"node-fetch": "^2.6.7",
|
|
@@ -1,21 +1,51 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
|
+
import { z } from 'zod';
|
|
2
3
|
import { UnifiedConfig } from '../../types';
|
|
3
|
-
|
|
4
|
-
addPlugins
|
|
4
|
+
declare const RedteamGenerateOptionsSchema: z.ZodObject<{
|
|
5
|
+
addPlugins: z.ZodOptional<z.ZodArray<z.ZodEnum<[string, ...string[]]>, "many">>;
|
|
6
|
+
cache: z.ZodBoolean;
|
|
7
|
+
config: z.ZodOptional<z.ZodString>;
|
|
8
|
+
defaultConfig: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
9
|
+
defaultConfigPath: z.ZodOptional<z.ZodString>;
|
|
10
|
+
envFile: z.ZodOptional<z.ZodString>;
|
|
11
|
+
injectVar: z.ZodOptional<z.ZodString>;
|
|
12
|
+
numTests: z.ZodNumber;
|
|
13
|
+
output: z.ZodOptional<z.ZodString>;
|
|
14
|
+
plugins: z.ZodOptional<z.ZodArray<z.ZodEnum<[string, ...string[]]>, "many">>;
|
|
15
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
16
|
+
purpose: z.ZodOptional<z.ZodString>;
|
|
17
|
+
write: z.ZodBoolean;
|
|
18
|
+
}, "strip", z.ZodTypeAny, {
|
|
19
|
+
write: boolean;
|
|
5
20
|
cache: boolean;
|
|
6
|
-
config?: string;
|
|
7
|
-
envFile?: string;
|
|
8
|
-
injectVar?: string;
|
|
9
21
|
numTests: number;
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
22
|
+
defaultConfig: Record<string, unknown>;
|
|
23
|
+
output?: string | undefined;
|
|
24
|
+
config?: string | undefined;
|
|
25
|
+
envFile?: string | undefined;
|
|
26
|
+
provider?: string | undefined;
|
|
27
|
+
purpose?: string | undefined;
|
|
28
|
+
injectVar?: string | undefined;
|
|
29
|
+
plugins?: string[] | undefined;
|
|
30
|
+
addPlugins?: string[] | undefined;
|
|
31
|
+
defaultConfigPath?: string | undefined;
|
|
32
|
+
}, {
|
|
14
33
|
write: boolean;
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
34
|
+
cache: boolean;
|
|
35
|
+
numTests: number;
|
|
36
|
+
defaultConfig: Record<string, unknown>;
|
|
37
|
+
output?: string | undefined;
|
|
38
|
+
config?: string | undefined;
|
|
39
|
+
envFile?: string | undefined;
|
|
40
|
+
provider?: string | undefined;
|
|
41
|
+
purpose?: string | undefined;
|
|
42
|
+
injectVar?: string | undefined;
|
|
43
|
+
plugins?: string[] | undefined;
|
|
44
|
+
addPlugins?: string[] | undefined;
|
|
45
|
+
defaultConfigPath?: string | undefined;
|
|
46
|
+
}>;
|
|
47
|
+
type RedteamGenerateOptions = z.infer<typeof RedteamGenerateOptionsSchema>;
|
|
48
|
+
export declare function doGenerateRedteam(options: RedteamGenerateOptions): Promise<void>;
|
|
19
49
|
export declare function generateRedteamCommand(program: Command, defaultConfig: Partial<UnifiedConfig>, defaultConfigPath: string | undefined): void;
|
|
20
50
|
export {};
|
|
21
51
|
//# sourceMappingURL=redteam.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../../src/commands/generate/redteam.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../../src/commands/generate/redteam.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAYxB,OAAO,EAAa,aAAa,EAAE,MAAM,aAAa,CAAC;AAGvD,QAAA,MAAM,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAchC,CAAC;AAEH,KAAK,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAE3E,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,sBAAsB,iBA4FtE;AAED,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,OAAO,EAChB,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,EACrC,iBAAiB,EAAE,MAAM,GAAG,SAAS,QAyDtC"}
|
|
@@ -32,6 +32,8 @@ const chalk_1 = __importDefault(require("chalk"));
|
|
|
32
32
|
const dedent_1 = __importDefault(require("dedent"));
|
|
33
33
|
const fs = __importStar(require("fs"));
|
|
34
34
|
const js_yaml_1 = __importDefault(require("js-yaml"));
|
|
35
|
+
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
36
|
+
const zod_1 = require("zod");
|
|
35
37
|
const cache_1 = require("../../cache");
|
|
36
38
|
const config_1 = require("../../config");
|
|
37
39
|
const logger_1 = __importDefault(require("../../logger"));
|
|
@@ -39,21 +41,36 @@ const redteam_1 = require("../../redteam");
|
|
|
39
41
|
const constants_1 = require("../../redteam/constants");
|
|
40
42
|
const telemetry_1 = __importDefault(require("../../telemetry"));
|
|
41
43
|
const util_1 = require("../../util");
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
44
|
+
const RedteamGenerateOptionsSchema = zod_1.z.object({
|
|
45
|
+
addPlugins: zod_1.z.array(zod_1.z.enum(constants_1.ADDITIONAL_PLUGINS)).optional(),
|
|
46
|
+
cache: zod_1.z.boolean(),
|
|
47
|
+
config: zod_1.z.string().optional(),
|
|
48
|
+
defaultConfig: zod_1.z.record(zod_1.z.unknown()),
|
|
49
|
+
defaultConfigPath: zod_1.z.string().optional(),
|
|
50
|
+
envFile: zod_1.z.string().optional(),
|
|
51
|
+
injectVar: zod_1.z.string().optional(),
|
|
52
|
+
numTests: zod_1.z.number().int().positive(),
|
|
53
|
+
output: zod_1.z.string().optional(),
|
|
54
|
+
plugins: zod_1.z.array(zod_1.z.enum(constants_1.ALL_PLUGINS)).optional(),
|
|
55
|
+
provider: zod_1.z.string().optional(),
|
|
56
|
+
purpose: zod_1.z.string().optional(),
|
|
57
|
+
write: zod_1.z.boolean(),
|
|
58
|
+
});
|
|
59
|
+
async function doGenerateRedteam(options) {
|
|
60
|
+
(0, util_1.setupEnv)(options.envFile);
|
|
61
|
+
if (!options.cache) {
|
|
45
62
|
logger_1.default.info('Cache is disabled.');
|
|
46
63
|
(0, cache_1.disableCache)();
|
|
47
64
|
}
|
|
48
65
|
let testSuite;
|
|
49
|
-
const configPath = config || defaultConfigPath;
|
|
66
|
+
const configPath = options.config || options.defaultConfigPath;
|
|
50
67
|
if (configPath) {
|
|
51
68
|
const resolved = await (0, config_1.resolveConfigs)({
|
|
52
69
|
config: [configPath],
|
|
53
|
-
}, defaultConfig);
|
|
70
|
+
}, options.defaultConfig);
|
|
54
71
|
testSuite = resolved.testSuite;
|
|
55
72
|
}
|
|
56
|
-
else if (purpose) {
|
|
73
|
+
else if (options.purpose) {
|
|
57
74
|
// There is a purpose, so we can just have a dummy testsuite for standalone invocation
|
|
58
75
|
testSuite = {
|
|
59
76
|
prompts: [],
|
|
@@ -72,16 +89,21 @@ async function doGenerateRedteam({ addPlugins, cache, config, envFile, injectVar
|
|
|
72
89
|
numTestsExisting: (testSuite.tests || []).length,
|
|
73
90
|
});
|
|
74
91
|
await telemetry_1.default.send();
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
plugins
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
92
|
+
const defaultPlugins = Array.from(constants_1.DEFAULT_PLUGINS);
|
|
93
|
+
let plugins = options.plugins || defaultPlugins;
|
|
94
|
+
if (options.addPlugins && options.addPlugins.length > 0) {
|
|
95
|
+
plugins = [...new Set([...plugins, ...options.addPlugins])];
|
|
96
|
+
}
|
|
97
|
+
(0, tiny_invariant_1.default)(plugins && Array.isArray(plugins) && plugins.length > 0, 'No plugins found');
|
|
98
|
+
const redteamTests = await (0, redteam_1.synthesize)({
|
|
99
|
+
purpose: options.purpose,
|
|
100
|
+
injectVar: options.injectVar,
|
|
101
|
+
plugins,
|
|
102
|
+
provider: options.provider,
|
|
103
|
+
prompts: testSuite.prompts.map((prompt) => prompt.raw),
|
|
104
|
+
numTests: options.numTests,
|
|
83
105
|
});
|
|
84
|
-
if (output) {
|
|
106
|
+
if (options.output) {
|
|
85
107
|
const existingYaml = configPath
|
|
86
108
|
? js_yaml_1.default.load(fs.readFileSync(configPath, 'utf8'))
|
|
87
109
|
: {};
|
|
@@ -93,12 +115,12 @@ async function doGenerateRedteam({ addPlugins, cache, config, envFile, injectVar
|
|
|
93
115
|
redteam: true,
|
|
94
116
|
},
|
|
95
117
|
};
|
|
96
|
-
fs.writeFileSync(output, js_yaml_1.default.dump(updatedYaml, { skipInvalid: true }));
|
|
118
|
+
fs.writeFileSync(options.output, js_yaml_1.default.dump(updatedYaml, { skipInvalid: true }));
|
|
97
119
|
(0, util_1.printBorder)();
|
|
98
|
-
logger_1.default.info(`Wrote ${redteamTests.length} new test cases to ${output}`);
|
|
120
|
+
logger_1.default.info(`Wrote ${redteamTests.length} new test cases to ${options.output}`);
|
|
99
121
|
(0, util_1.printBorder)();
|
|
100
122
|
}
|
|
101
|
-
else if (write && configPath) {
|
|
123
|
+
else if (options.write && configPath) {
|
|
102
124
|
const existingConfig = js_yaml_1.default.load(fs.readFileSync(configPath, 'utf8'));
|
|
103
125
|
existingConfig.tests = [...(existingConfig.tests || []), ...redteamTests];
|
|
104
126
|
fs.writeFileSync(configPath, js_yaml_1.default.dump(existingConfig));
|
|
@@ -109,11 +131,11 @@ async function doGenerateRedteam({ addPlugins, cache, config, envFile, injectVar
|
|
|
109
131
|
logger_1.default.info(js_yaml_1.default.dump(redteamTests, { skipInvalid: true }));
|
|
110
132
|
}
|
|
111
133
|
telemetry_1.default.record('command_used', {
|
|
134
|
+
duration: Math.round((Date.now() - startTime) / 1000),
|
|
112
135
|
name: 'generate redteam',
|
|
113
136
|
numPrompts: testSuite.prompts.length,
|
|
114
137
|
numTestsExisting: (testSuite.tests || []).length,
|
|
115
138
|
numTestsGenerated: redteamTests.length,
|
|
116
|
-
duration: Math.round((Date.now() - startTime) / 1000),
|
|
117
139
|
});
|
|
118
140
|
await telemetry_1.default.send();
|
|
119
141
|
}
|
|
@@ -123,7 +145,7 @@ function generateRedteamCommand(program, defaultConfig, defaultConfigPath) {
|
|
|
123
145
|
.description('Generate adversarial test cases')
|
|
124
146
|
.option('-c, --config [path]', 'Path to configuration file. Defaults to promptfooconfig.yaml')
|
|
125
147
|
.option('-o, --output [path]', 'Path to output file')
|
|
126
|
-
.option('-w, --write', 'Write results to promptfoo configuration file')
|
|
148
|
+
.option('-w, --write', 'Write results to promptfoo configuration file', false)
|
|
127
149
|
.option('--purpose <purpose>', 'Set the system purpose. If not set, the system purpose will be inferred from the config file')
|
|
128
150
|
.option('--provider <provider>', `Provider to use for generating adversarial tests. Defaults to: ${constants_1.REDTEAM_MODEL}`)
|
|
129
151
|
.option('--injectVar <varname>', 'Override the {{variable}} that represents user input in the prompt. Default value is inferred from your prompts')
|
|
@@ -133,11 +155,30 @@ function generateRedteamCommand(program, defaultConfig, defaultConfigPath) {
|
|
|
133
155
|
.option('--add-plugins <plugins>', (0, dedent_1.default) `Comma-separated list of plugins to run in addition to the default plugins:
|
|
134
156
|
\n- ${constants_1.ADDITIONAL_PLUGINS.sort().join('\n- ')}\n\n
|
|
135
157
|
`, (val) => val.split(',').map((x) => x.trim()))
|
|
136
|
-
.option('--no-cache', 'Do not read or write results to disk cache', false)
|
|
137
158
|
.option('-n, --num-tests <number>', 'Number of test cases to generate per plugin', parseInt, 5)
|
|
159
|
+
.option('--no-cache', 'Do not read or write results to disk cache', false)
|
|
138
160
|
.option('--env-file <path>', 'Path to .env file')
|
|
139
161
|
.action((opts) => {
|
|
140
|
-
|
|
162
|
+
try {
|
|
163
|
+
const validatedOpts = RedteamGenerateOptionsSchema.parse({
|
|
164
|
+
...opts,
|
|
165
|
+
defaultConfig,
|
|
166
|
+
defaultConfigPath,
|
|
167
|
+
});
|
|
168
|
+
doGenerateRedteam(validatedOpts);
|
|
169
|
+
}
|
|
170
|
+
catch (error) {
|
|
171
|
+
if (error instanceof zod_1.z.ZodError) {
|
|
172
|
+
logger_1.default.error('Invalid options:');
|
|
173
|
+
error.errors.forEach((err) => {
|
|
174
|
+
logger_1.default.error(` ${err.path.join('.')}: ${err.message}`);
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
else {
|
|
178
|
+
logger_1.default.error('An unexpected error occurred:', error);
|
|
179
|
+
}
|
|
180
|
+
process.exit(1);
|
|
181
|
+
}
|
|
141
182
|
});
|
|
142
183
|
}
|
|
143
184
|
//# sourceMappingURL=redteam.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redteam.js","sourceRoot":"","sources":["../../../../src/commands/generate/redteam.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"redteam.js","sourceRoot":"","sources":["../../../../src/commands/generate/redteam.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuCA,8CA4FC;AAED,wDA4DC;AAjMD,kDAA0B;AAE1B,oDAA4B;AAC5B,uCAAyB;AACzB,sDAA2B;AAC3B,oEAAuC;AACvC,6BAAwB;AACxB,uCAA2C;AAC3C,yCAA8C;AAC9C,0DAAkC;AAClC,2CAA2C;AAC3C,uDAKiC;AACjC,gEAAwC;AAExC,qCAAmD;AAEnD,MAAM,4BAA4B,GAAG,OAAC,CAAC,MAAM,CAAC;IAC5C,UAAU,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,IAAI,CAAC,8BAA0B,CAAC,CAAC,CAAC,QAAQ,EAAE;IAClE,KAAK,EAAE,OAAC,CAAC,OAAO,EAAE;IAClB,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,aAAa,EAAE,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,OAAO,EAAE,CAAC;IACpC,iBAAiB,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACxC,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,SAAS,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACrC,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,OAAO,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,IAAI,CAAC,uBAAmB,CAAC,CAAC,CAAC,QAAQ,EAAE;IACxD,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,KAAK,EAAE,OAAC,CAAC,OAAO,EAAE;CACnB,CAAC,CAAC;AAII,KAAK,UAAU,iBAAiB,CAAC,OAA+B;IACrE,IAAA,eAAQ,EAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1B,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,gBAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAClC,IAAA,oBAAY,GAAE,CAAC;IACjB,CAAC;IAED,IAAI,SAAoB,CAAC;IACzB,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAC/D,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,QAAQ,GAAG,MAAM,IAAA,uBAAc,EACnC;YACE,MAAM,EAAE,CAAC,UAAU,CAAC;SACrB,EACD,OAAO,CAAC,aAAa,CACtB,CAAC;QACF,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC;IACjC,CAAC;SAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,sFAAsF;QACtF,SAAS,GAAG;YACV,OAAO,EAAE,EAAE;YACX,SAAS,EAAE,EAAE;YACb,KAAK,EAAE,EAAE;SACV,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,GAAG,CACP,gDAAgD,eAAK,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CACpG,CACF,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QAC/B,IAAI,EAAE,4BAA4B;QAClC,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM;QACpC,gBAAgB,EAAE,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM;KACjD,CAAC,CAAC;IACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;IAEvB,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,2BAAuB,CAAC,CAAC;IAC3D,IAAI,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,cAAc,CAAC;IAChD,IAAI,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxD,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC9D,CAAC;IACD,IAAA,wBAAS,EAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,kBAAkB,CAAC,CAAC;IACvF,MAAM,YAAY,GAAG,MAAM,IAAA,oBAAU,EAAC;QACpC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,OAAO;QACP,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC;QACtD,QAAQ,EAAE,OAAO,CAAC,QAAQ;KAC3B,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,YAAY,GAAG,UAAU;YAC7B,CAAC,CAAE,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAA4B;YAC5E,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,WAAW,GAA2B;YAC1C,GAAG,YAAY;YACf,KAAK,EAAE,YAAY;YACnB,QAAQ,EAAE;gBACR,GAAG,YAAY,CAAC,QAAQ;gBACxB,OAAO,EAAE,IAAI;aACd;SACF,CAAC;QACF,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,iBAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAChF,IAAA,kBAAW,GAAE,CAAC;QACd,gBAAM,CAAC,IAAI,CAAC,SAAS,YAAY,CAAC,MAAM,sBAAsB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAChF,IAAA,kBAAW,GAAE,CAAC;IAChB,CAAC;SAAM,IAAI,OAAO,CAAC,KAAK,IAAI,UAAU,EAAE,CAAC;QACvC,MAAM,cAAc,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAA2B,CAAC;QAChG,cAAc,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,GAAG,YAAY,CAAC,CAAC;QAC1E,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,iBAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;QACxD,gBAAM,CAAC,IAAI,CAAC,WAAW,YAAY,CAAC,MAAM,sBAAsB,UAAU,EAAE,CAAC,CAAC;QAC9E,gBAAM,CAAC,IAAI,CACT,IAAI,GAAG,eAAK,CAAC,KAAK,CAAC,OAAO,eAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,6BAA6B,CAAC,CACrF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,gBAAM,CAAC,IAAI,CAAC,iBAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QAC/B,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;QACrD,IAAI,EAAE,kBAAkB;QACxB,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM;QACpC,gBAAgB,EAAE,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM;QAChD,iBAAiB,EAAE,YAAY,CAAC,MAAM;KACvC,CAAC,CAAC;IACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAED,SAAgB,sBAAsB,CACpC,OAAgB,EAChB,aAAqC,EACrC,iBAAqC;IAErC,OAAO;SACJ,OAAO,CAAC,SAAS,CAAC;SAClB,WAAW,CAAC,iCAAiC,CAAC;SAC9C,MAAM,CAAC,qBAAqB,EAAE,8DAA8D,CAAC;SAC7F,MAAM,CAAC,qBAAqB,EAAE,qBAAqB,CAAC;SACpD,MAAM,CAAC,aAAa,EAAE,+CAA+C,EAAE,KAAK,CAAC;SAC7E,MAAM,CACL,qBAAqB,EACrB,8FAA8F,CAC/F;SACA,MAAM,CACL,uBAAuB,EACvB,kEAAkE,yBAAa,EAAE,CAClF;SACA,MAAM,CACL,uBAAuB,EACvB,iHAAiH,CAClH;SACA,MAAM,CACL,qBAAqB,EACrB,IAAA,gBAAM,EAAA;cACE,KAAK,CAAC,IAAI,CAAC,2BAAuB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;KAChE,EACC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAC7C;SACA,MAAM,CACL,yBAAyB,EACzB,IAAA,gBAAM,EAAA;cACE,8BAA0B,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;OACrD,EACD,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAC7C;SACA,MAAM,CAAC,0BAA0B,EAAE,6CAA6C,EAAE,QAAQ,EAAE,CAAC,CAAC;SAC9F,MAAM,CAAC,YAAY,EAAE,4CAA4C,EAAE,KAAK,CAAC;SACzE,MAAM,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;SAChD,MAAM,CAAC,CAAC,IAAqC,EAAQ,EAAE;QACtD,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,4BAA4B,CAAC,KAAK,CAAC;gBACvD,GAAG,IAAI;gBACP,aAAa;gBACb,iBAAiB;aAClB,CAAC,CAAC;YACH,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,OAAC,CAAC,QAAQ,EAAE,CAAC;gBAChC,gBAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAe,EAAE,EAAE;oBACvC,gBAAM,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC1D,CAAC,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,gBAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;YACvD,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -1,10 +1,3 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
|
-
interface RunRedteamOptions {
|
|
3
|
-
config: string;
|
|
4
|
-
cache: boolean;
|
|
5
|
-
envFile: string;
|
|
6
|
-
}
|
|
7
|
-
export declare function doRunRedteam(cmdObj: RunRedteamOptions): Promise<void>;
|
|
8
2
|
export declare function redteamCommand(program: Command): void;
|
|
9
|
-
export {};
|
|
10
3
|
//# sourceMappingURL=redteam.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/commands/redteam.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/commands/redteam.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,QAwJ9C"}
|
|
@@ -26,18 +26,21 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
26
26
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
27
|
};
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.doRunRedteam = doRunRedteam;
|
|
30
29
|
exports.redteamCommand = redteamCommand;
|
|
30
|
+
const checkbox_1 = __importDefault(require("@inquirer/checkbox"));
|
|
31
|
+
const confirm_1 = __importDefault(require("@inquirer/confirm"));
|
|
32
|
+
const editor_1 = __importDefault(require("@inquirer/editor"));
|
|
33
|
+
const input_1 = __importDefault(require("@inquirer/input"));
|
|
34
|
+
const number_1 = __importDefault(require("@inquirer/number"));
|
|
35
|
+
const rawlist_1 = __importDefault(require("@inquirer/rawlist"));
|
|
31
36
|
const chalk_1 = __importDefault(require("chalk"));
|
|
32
37
|
const fs = __importStar(require("fs"));
|
|
33
|
-
const inquirer_1 = __importDefault(require("inquirer"));
|
|
34
38
|
const js_yaml_1 = __importDefault(require("js-yaml"));
|
|
35
39
|
const path = __importStar(require("path"));
|
|
36
40
|
const logger_1 = __importDefault(require("../logger"));
|
|
37
41
|
const constants_1 = require("../redteam/constants");
|
|
38
42
|
const telemetry_1 = __importDefault(require("../telemetry"));
|
|
39
43
|
const redteam_1 = require("./generate/redteam");
|
|
40
|
-
async function doRunRedteam(cmdObj) { }
|
|
41
44
|
function redteamCommand(program) {
|
|
42
45
|
const redteamCommand = program.command('redteam').description('Red team LLM applications');
|
|
43
46
|
redteamCommand
|
|
@@ -51,116 +54,103 @@ function redteamCommand(program) {
|
|
|
51
54
|
await telemetry_1.default.send();
|
|
52
55
|
let projectDir = directory;
|
|
53
56
|
if (!projectDir) {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
message: 'Where do you want to create the project?',
|
|
59
|
-
default: '.',
|
|
60
|
-
},
|
|
61
|
-
]);
|
|
62
|
-
projectDir = chosenDir || '.';
|
|
57
|
+
projectDir = await (0, input_1.default)({
|
|
58
|
+
message: 'Where do you want to create the project?',
|
|
59
|
+
default: '.',
|
|
60
|
+
});
|
|
63
61
|
}
|
|
64
62
|
if (projectDir !== '.' && !fs.existsSync(projectDir)) {
|
|
65
63
|
fs.mkdirSync(projectDir, { recursive: true });
|
|
66
64
|
}
|
|
65
|
+
const configPath = path.join(projectDir, 'promptfooconfig.yaml');
|
|
66
|
+
if (fs.existsSync(configPath)) {
|
|
67
|
+
const overwrite = await (0, confirm_1.default)({
|
|
68
|
+
message: 'A promptfoo configuration file already exists. Do you want to overwrite it?',
|
|
69
|
+
default: false,
|
|
70
|
+
});
|
|
71
|
+
if (!overwrite) {
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
67
75
|
// Question 2: Prompt
|
|
68
|
-
const
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
name: '
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
{ name: 'Reference a prompt file', value: 'file' },
|
|
76
|
-
],
|
|
77
|
-
},
|
|
78
|
-
]);
|
|
76
|
+
const promptChoice = await (0, rawlist_1.default)({
|
|
77
|
+
message: 'How would you like to specify the prompt?',
|
|
78
|
+
choices: [
|
|
79
|
+
{ name: 'Enter a prompt', value: 'enter' },
|
|
80
|
+
{ name: 'Reference a prompt file', value: 'file' },
|
|
81
|
+
],
|
|
82
|
+
});
|
|
79
83
|
let prompt;
|
|
80
84
|
if (promptChoice === 'enter') {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
message: 'Enter your prompt:',
|
|
86
|
-
default: 'You are a helpful concise assistant.\n\nUser query: {{query}}\n\n(NOTE: your prompt must include "{{query}}" as a placeholder for user input)',
|
|
87
|
-
},
|
|
88
|
-
]);
|
|
89
|
-
prompt = enteredPrompt;
|
|
85
|
+
prompt = await (0, editor_1.default)({
|
|
86
|
+
message: 'Enter your prompt:',
|
|
87
|
+
default: 'You are a helpful concise assistant.\n\nUser query: {{query}}\n\n(NOTE: your prompt must include "{{query}}" as a placeholder for user input)',
|
|
88
|
+
});
|
|
90
89
|
}
|
|
91
90
|
else {
|
|
92
|
-
const
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
name: 'promptFile',
|
|
96
|
-
message: 'Enter the path to your prompt file (text or JSON):',
|
|
97
|
-
},
|
|
98
|
-
]);
|
|
91
|
+
const promptFile = await (0, input_1.default)({
|
|
92
|
+
message: 'Enter the path to your prompt file (text or JSON):',
|
|
93
|
+
});
|
|
99
94
|
prompt = `file://${promptFile}`;
|
|
100
95
|
}
|
|
101
96
|
// Question 3: Provider
|
|
102
97
|
let provider;
|
|
103
|
-
const
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
name: '
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
'
|
|
110
|
-
'
|
|
111
|
-
|
|
112
|
-
'anthropic:messages:claude-3-opus-20240307',
|
|
113
|
-
'vertex:gemini-pro',
|
|
114
|
-
'Other',
|
|
115
|
-
],
|
|
116
|
-
},
|
|
117
|
-
]);
|
|
118
|
-
if (providerChoice === 'Other') {
|
|
119
|
-
const { customProvider } = await inquirer_1.default.prompt([
|
|
98
|
+
const providerChoice = await (0, rawlist_1.default)({
|
|
99
|
+
message: 'Choose a provider:',
|
|
100
|
+
choices: [
|
|
101
|
+
{ name: 'openai:gpt-3.5-turbo', value: 'openai:gpt-3.5-turbo' },
|
|
102
|
+
{ name: 'openai:gpt-4o', value: 'openai:gpt-4o' },
|
|
103
|
+
{
|
|
104
|
+
name: 'anthropic:messages:claude-3-5-sonnet-20240620',
|
|
105
|
+
value: 'anthropic:messages:claude-3-5-sonnet-20240620',
|
|
106
|
+
},
|
|
120
107
|
{
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
message: 'Enter the provider ID (see https://www.promptfoo.dev/docs/providers/ for options):',
|
|
108
|
+
name: 'anthropic:claude-3-opus-20240307',
|
|
109
|
+
value: 'anthropic:messages:claude-3-opus-20240307',
|
|
124
110
|
},
|
|
125
|
-
|
|
126
|
-
|
|
111
|
+
{ name: 'vertex:gemini-pro', value: 'vertex:gemini-pro' },
|
|
112
|
+
{ name: 'Other', value: 'Other' },
|
|
113
|
+
],
|
|
114
|
+
});
|
|
115
|
+
if (providerChoice === 'Other') {
|
|
116
|
+
provider = await (0, input_1.default)({
|
|
117
|
+
message: 'Enter the provider ID (see https://www.promptfoo.dev/docs/providers/ for options):',
|
|
118
|
+
});
|
|
127
119
|
}
|
|
128
120
|
else {
|
|
129
121
|
provider = providerChoice;
|
|
130
122
|
}
|
|
131
123
|
// Question 4: Plugins
|
|
132
|
-
const pluginChoices = Array.from(constants_1.ALL_PLUGINS)
|
|
124
|
+
const pluginChoices = Array.from(constants_1.ALL_PLUGINS)
|
|
125
|
+
.sort()
|
|
126
|
+
.map((plugin) => ({
|
|
133
127
|
name: `${plugin} - ${constants_1.subCategoryDescriptions[plugin] || 'No description available'}`,
|
|
134
128
|
value: plugin,
|
|
135
129
|
checked: constants_1.DEFAULT_PLUGINS.has(plugin),
|
|
136
130
|
}));
|
|
137
|
-
const
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
131
|
+
const plugins = await (0, checkbox_1.default)({
|
|
132
|
+
message: 'Select the plugins you want to enable:',
|
|
133
|
+
choices: pluginChoices,
|
|
134
|
+
pageSize: 20,
|
|
135
|
+
});
|
|
136
|
+
const numTests = (await (0, number_1.default)({
|
|
137
|
+
message: 'How many test cases do you want to generate per plugin?',
|
|
138
|
+
default: 5,
|
|
139
|
+
min: 0,
|
|
140
|
+
max: 1000,
|
|
141
|
+
})) ?? 5;
|
|
147
142
|
// Create config file
|
|
148
143
|
const config = {
|
|
149
144
|
prompts: [prompt],
|
|
150
145
|
providers: [provider],
|
|
151
146
|
tests: [],
|
|
152
147
|
};
|
|
153
|
-
const configPath = path.join(projectDir, 'promptfooconfig.yaml');
|
|
154
148
|
fs.writeFileSync(configPath, js_yaml_1.default.dump(config), 'utf8');
|
|
155
149
|
logger_1.default.info('\n' + chalk_1.default.green(`Created red teaming configuration file at ${configPath}`) + '\n');
|
|
156
|
-
const
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
message: 'Are you ready to generate adversarial test cases?',
|
|
161
|
-
default: true,
|
|
162
|
-
},
|
|
163
|
-
]);
|
|
150
|
+
const readyToGenerate = await (0, confirm_1.default)({
|
|
151
|
+
message: 'Are you ready to generate adversarial test cases?',
|
|
152
|
+
default: true,
|
|
153
|
+
});
|
|
164
154
|
telemetry_1.default.record('command_used', {
|
|
165
155
|
name: 'redteam init',
|
|
166
156
|
});
|
|
@@ -172,7 +162,7 @@ function redteamCommand(program) {
|
|
|
172
162
|
write: true,
|
|
173
163
|
defaultConfig: config,
|
|
174
164
|
defaultConfigPath: configPath,
|
|
175
|
-
numTests
|
|
165
|
+
numTests,
|
|
176
166
|
});
|
|
177
167
|
}
|
|
178
168
|
else {
|
|
@@ -181,12 +171,5 @@ function redteamCommand(program) {
|
|
|
181
171
|
chalk_1.default.bold('promptfoo generate redteam')));
|
|
182
172
|
}
|
|
183
173
|
});
|
|
184
|
-
redteamCommand
|
|
185
|
-
.command('run')
|
|
186
|
-
.description('Run red teaming evaluation')
|
|
187
|
-
.option('-c, --config <path>', 'Path to configuration file')
|
|
188
|
-
.option('--no-cache', 'Disable cache', false)
|
|
189
|
-
.option('--env-file <path>', 'Path to .env file')
|
|
190
|
-
.action(doRunRedteam);
|
|
191
174
|
}
|
|
192
175
|
//# sourceMappingURL=redteam.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redteam.js","sourceRoot":"","sources":["../../../src/commands/redteam.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"redteam.js","sourceRoot":"","sources":["../../../src/commands/redteam.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgBA,wCAwJC;AAxKD,kEAA0C;AAC1C,gEAAwC;AACxC,8DAAsC;AACtC,4DAAoC;AACpC,8DAAsC;AACtC,gEAAwC;AACxC,kDAA0B;AAE1B,uCAAyB;AACzB,sDAA2B;AAC3B,2CAA6B;AAC7B,uDAA+B;AAC/B,oDAA6F;AAC7F,6DAAqC;AACrC,gDAAuD;AAEvD,SAAgB,cAAc,CAAC,OAAgB;IAC7C,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,WAAW,CAAC,2BAA2B,CAAC,CAAC;IAE3F,cAAc;SACX,OAAO,CAAC,kBAAkB,CAAC;SAC3B,WAAW,CAAC,gCAAgC,CAAC;SAC7C,MAAM,CAAC,KAAK,EAAE,SAA6B,EAAE,EAAE;QAC9C,mBAAS,CAAC,eAAe,EAAE,CAAC;QAC5B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,wBAAwB;SAC/B,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,IAAI,UAAU,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,UAAU,GAAG,MAAM,IAAA,eAAK,EAAC;gBACvB,OAAO,EAAE,0CAA0C;gBACnD,OAAO,EAAE,GAAG;aACb,CAAC,CAAC;QACL,CAAC;QACD,IAAI,UAAU,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YACrD,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAC;QACjE,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,MAAM,SAAS,GAAG,MAAM,IAAA,iBAAO,EAAC;gBAC9B,OAAO,EAAE,6EAA6E;gBACtF,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,OAAO;YACT,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,MAAM,YAAY,GAAG,MAAM,IAAA,iBAAO,EAAC;YACjC,OAAO,EAAE,2CAA2C;YACpD,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,OAAO,EAAE;gBAC1C,EAAE,IAAI,EAAE,yBAAyB,EAAE,KAAK,EAAE,MAAM,EAAE;aACnD;SACF,CAAC,CAAC;QAEH,IAAI,MAAc,CAAC;QACnB,IAAI,YAAY,KAAK,OAAO,EAAE,CAAC;YAC7B,MAAM,GAAG,MAAM,IAAA,gBAAM,EAAC;gBACpB,OAAO,EAAE,oBAAoB;gBAC7B,OAAO,EACL,+IAA+I;aAClJ,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,UAAU,GAAG,MAAM,IAAA,eAAK,EAAC;gBAC7B,OAAO,EAAE,oDAAoD;aAC9D,CAAC,CAAC;YACH,MAAM,GAAG,UAAU,UAAU,EAAE,CAAC;QAClC,CAAC;QAED,uBAAuB;QACvB,IAAI,QAAgB,CAAC;QACrB,MAAM,cAAc,GAAG,MAAM,IAAA,iBAAO,EAAC;YACnC,OAAO,EAAE,oBAAoB;YAC7B,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,sBAAsB,EAAE,KAAK,EAAE,sBAAsB,EAAE;gBAC/D,EAAE,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,eAAe,EAAE;gBACjD;oBACE,IAAI,EAAE,+CAA+C;oBACrD,KAAK,EAAE,+CAA+C;iBACvD;gBACD;oBACE,IAAI,EAAE,kCAAkC;oBACxC,KAAK,EAAE,2CAA2C;iBACnD;gBACD,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,mBAAmB,EAAE;gBACzD,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE;aAClC;SACF,CAAC,CAAC;QAEH,IAAI,cAAc,KAAK,OAAO,EAAE,CAAC;YAC/B,QAAQ,GAAG,MAAM,IAAA,eAAK,EAAC;gBACrB,OAAO,EACL,oFAAoF;aACvF,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,cAAc,CAAC;QAC5B,CAAC;QAED,sBAAsB;QACtB,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,uBAAW,CAAC;aAC1C,IAAI,EAAE;aACN,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAChB,IAAI,EAAE,GAAG,MAAM,MAAM,mCAAuB,CAAC,MAAM,CAAC,IAAI,0BAA0B,EAAE;YACpF,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,2BAAe,CAAC,GAAG,CAAC,MAAM,CAAC;SACrC,CAAC,CAAC,CAAC;QAEN,MAAM,OAAO,GAAG,MAAM,IAAA,kBAAQ,EAAC;YAC7B,OAAO,EAAE,wCAAwC;YACjD,OAAO,EAAE,aAAa;YACtB,QAAQ,EAAE,EAAE;SACb,CAAC,CAAC;QAEH,MAAM,QAAQ,GACZ,CAAC,MAAM,IAAA,gBAAM,EAAC;YACZ,OAAO,EAAE,yDAAyD;YAClE,OAAO,EAAE,CAAC;YACV,GAAG,EAAE,CAAC;YACN,GAAG,EAAE,IAAI;SACV,CAAC,CAAC,IAAI,CAAC,CAAC;QAEX,qBAAqB;QACrB,MAAM,MAAM,GAAG;YACb,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,SAAS,EAAE,CAAC,QAAQ,CAAC;YACrB,KAAK,EAAE,EAAE;SACV,CAAC;QAEF,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,iBAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QAExD,gBAAM,CAAC,IAAI,CACT,IAAI,GAAG,eAAK,CAAC,KAAK,CAAC,6CAA6C,UAAU,EAAE,CAAC,GAAG,IAAI,CACrF,CAAC;QAEF,MAAM,eAAe,GAAG,MAAM,IAAA,iBAAO,EAAC;YACpC,OAAO,EAAE,mDAAmD;YAC5D,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;QAEH,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,cAAc;SACrB,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,IAAA,2BAAiB,EAAC;gBACtB,OAAO;gBACP,KAAK,EAAE,KAAK;gBACZ,KAAK,EAAE,IAAI;gBACX,aAAa,EAAE,MAAM;gBACrB,iBAAiB,EAAE,UAAU;gBAC7B,QAAQ;aACT,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,gBAAM,CAAC,IAAI,CACT,IAAI;gBACF,eAAK,CAAC,IAAI,CACR,iDAAiD;oBAC/C,eAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAC3C,CACJ,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"share.d.ts","sourceRoot":"","sources":["../../../src/commands/share.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQpC,wBAAgB,YAAY,CAAC,OAAO,EAAE,OAAO,
|
|
1
|
+
{"version":3,"file":"share.d.ts","sourceRoot":"","sources":["../../../src/commands/share.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQpC,wBAAgB,YAAY,CAAC,OAAO,EAAE,OAAO,QAkD5C"}
|
|
@@ -41,7 +41,10 @@ function shareCommand(program) {
|
|
|
41
41
|
input: process.stdin,
|
|
42
42
|
output: process.stdout,
|
|
43
43
|
});
|
|
44
|
-
|
|
44
|
+
const hostname = process.env.PROMPTFOO_SHARING_APP_BASE_URL
|
|
45
|
+
? new URL(process.env.PROMPTFOO_SHARING_APP_BASE_URL).hostname
|
|
46
|
+
: 'app.promptfoo.dev';
|
|
47
|
+
reader.question(`Create a private shareable URL of your most recent eval on ${hostname}?\n\nTo proceed, please confirm [Y/n] `, async function (answer) {
|
|
45
48
|
if (answer.toLowerCase() !== 'yes' && answer.toLowerCase() !== 'y' && answer !== '') {
|
|
46
49
|
reader.close();
|
|
47
50
|
process.exit(1);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"share.js","sourceRoot":"","sources":["../../../src/commands/share.ts"],"names":[],"mappings":";;;;;AASA,
|
|
1
|
+
{"version":3,"file":"share.js","sourceRoot":"","sources":["../../../src/commands/share.ts"],"names":[],"mappings":";;;;;AASA,oCAkDC;AA3DD,kDAA0B;AAE1B,wDAAgC;AAChC,uDAA+B;AAC/B,oCAA8C;AAC9C,6DAAqC;AACrC,kCAAmC;AACnC,kCAA4C;AAE5C,SAAgB,YAAY,CAAC,OAAgB;IAC3C,OAAO;SACJ,OAAO,CAAC,OAAO,CAAC;SAChB,WAAW,CAAC,iDAAiD,CAAC;SAC9D,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC;SACxC,MAAM,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;SAChD,MAAM,CAAC,KAAK,EAAE,MAAoD,EAAE,EAAE;QACrE,IAAA,eAAQ,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzB,mBAAS,CAAC,eAAe,EAAE,CAAC;QAC5B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,OAAO;SACd,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,MAAM,eAAe,GAAG,KAAK,IAAI,EAAE;YACjC,MAAM,aAAa,GAAG,MAAM,IAAA,wBAAiB,GAAE,CAAC;YAChD,IAAI,CAAC,aAAa,EAAE,CAAC;gBACnB,gBAAM,CAAC,KAAK,CAAC,oEAAoE,CAAC,CAAC;gBACnF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YACD,MAAM,GAAG,GAAG,MAAM,IAAA,0BAAkB,EAAC,aAAa,CAAC,OAAO,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;YAClF,gBAAM,CAAC,IAAI,CAAC,iBAAiB,eAAK,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9D,CAAC,CAAC;QAEF,IAAI,MAAM,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,+BAA+B,EAAE,CAAC;YAC9D,eAAe,EAAE,CAAC;QACpB,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,kBAAQ,CAAC,eAAe,CAAC;gBACtC,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;aACvB,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,8BAA8B;gBACzD,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC,QAAQ;gBAC9D,CAAC,CAAC,mBAAmB,CAAC;YAExB,MAAM,CAAC,QAAQ,CACb,8DAA8D,QAAQ,wCAAwC,EAC9G,KAAK,WAAW,MAAc;gBAC5B,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,GAAG,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;oBACpF,MAAM,CAAC,KAAK,EAAE,CAAC;oBACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAClB,CAAC;gBACD,MAAM,CAAC,KAAK,EAAE,CAAC;gBAEf,eAAe,EAAE,CAAC;YACpB,CAAC,CACF,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}
|