promptfoo 0.69.1 → 0.70.0

package/dist/src/commands/eval.js

@@ -0,0 +1,280 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.doEval = doEval;
+exports.evalCommand = evalCommand;
+const chalk_1 = __importDefault(require("chalk"));
+const chokidar_1 = __importDefault(require("chokidar"));
+const path = __importStar(require("path"));
+const cache_1 = require("../cache");
+const cliState_1 = __importDefault(require("../cliState"));
+const config_1 = require("../config");
+const evaluator_1 = require("../evaluator");
+const logger_1 = __importStar(require("../logger"));
+const providers_1 = require("../providers");
+const share_1 = require("../share");
+const table_1 = require("../table");
+const telemetry_1 = __importDefault(require("../telemetry"));
+const util_1 = require("../util");
+const filterProviders_1 = require("./eval/filterProviders");
+const filterTests_1 = require("./eval/filterTests");
+async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions) {
+    (0, util_1.setupEnv)(cmdObj.envFile);
+    let config = undefined;
+    let testSuite = undefined;
+    let basePath = undefined;
+    const runEvaluation = async (initialization) => {
+        const startTime = Date.now();
+        telemetry_1.default.record('command_used', {
+            name: 'eval - started',
+            watch: Boolean(cmdObj.watch),
+        });
+        await telemetry_1.default.send();
+        // Misc settings
+        if (cmdObj.verbose) {
+            (0, logger_1.setLogLevel)('debug');
+        }
+        const iterations = parseInt(cmdObj.repeat || '', 10);
+        const repeat = !isNaN(iterations) && iterations > 0 ? iterations : 1;
+        if (!cmdObj.cache || repeat > 1) {
+            logger_1.default.info('Cache is disabled.');
+            (0, cache_1.disableCache)();
+        }
+        ({ config, testSuite, basePath } = await (0, config_1.resolveConfigs)(cmdObj, defaultConfig));
+        cliState_1.default.basePath = basePath;
+        let maxConcurrency = parseInt(cmdObj.maxConcurrency || '', 10);
+        const delay = parseInt(cmdObj.delay || '', 0);
+        if (delay > 0) {
+            maxConcurrency = 1;
+            logger_1.default.info(`Running at concurrency=1 because ${delay}ms delay was requested between API calls`);
+        }
+        testSuite.tests = await (0, filterTests_1.filterTests)(testSuite, {
+            firstN: cmdObj.filterFirstN,
+            pattern: cmdObj.filterPattern,
+            failing: cmdObj.filterFailing,
+        });
+        testSuite.providers = (0, filterProviders_1.filterProviders)(testSuite.providers, cmdObj.filterProviders);
+        const options = {
+            showProgressBar: (0, logger_1.getLogLevel)() === 'debug' ? false : cmdObj.progressBar,
+            maxConcurrency: !isNaN(maxConcurrency) && maxConcurrency > 0 ? maxConcurrency : undefined,
+            repeat,
+            delay: !isNaN(delay) && delay > 0 ? delay : undefined,
+            interactiveProviders: cmdObj.interactiveProviders,
+            ...evaluateOptions,
+        };
+        if (cmdObj.grader) {
+            testSuite.defaultTest = testSuite.defaultTest || {};
+            testSuite.defaultTest.options = testSuite.defaultTest.options || {};
+            testSuite.defaultTest.options.provider = await (0, providers_1.loadApiProvider)(cmdObj.grader);
+        }
+        if (cmdObj.var) {
+            testSuite.defaultTest = testSuite.defaultTest || {};
+            testSuite.defaultTest.vars = { ...testSuite.defaultTest.vars, ...cmdObj.var };
+        }
+        if (cmdObj.generateSuggestions) {
+            options.generateSuggestions = true;
+        }
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {
+            ...options,
+            eventSource: 'cli',
+        });
+        const shareableUrl = cmdObj.share && config.sharing ? await (0, share_1.createShareableUrl)(summary, config) : null;
+        if (cmdObj.table && (0, logger_1.getLogLevel)() !== 'debug') {
+            // Output CLI table
+            const table = (0, table_1.generateTable)(summary, parseInt(cmdObj.tableCellMaxLength || '', 10));
+            logger_1.default.info('\n' + table.toString());
+            if (summary.table.body.length > 25) {
+                const rowsLeft = summary.table.body.length - 25;
+                logger_1.default.info(`... ${rowsLeft} more row${rowsLeft === 1 ? '' : 's'} not shown ...\n`);
+            }
+        }
+        else if (summary.stats.failures !== 0) {
+            logger_1.default.debug(`At least one evaluation failure occurred. This might be caused by the underlying call to the provider, or a test failure. Context: \n${JSON.stringify(summary.results)}`);
+        }
+        await (0, util_1.migrateResultsFromFileSystemToDatabase)();
+        let evalId = null;
+        if (cmdObj.write) {
+            evalId = await (0, util_1.writeResultsToDatabase)(summary, config);
+        }
+        const { outputPath } = config;
+        if (outputPath) {
+            // Write output to file
+            if (typeof outputPath === 'string') {
+                await (0, util_1.writeOutput)(outputPath, evalId, summary, config, shareableUrl);
+            }
+            else if (Array.isArray(outputPath)) {
+                await (0, util_1.writeMultipleOutputs)(outputPath, evalId, summary, config, shareableUrl);
+            }
+            logger_1.default.info(chalk_1.default.yellow(`Writing output to ${outputPath}`));
+        }
+        telemetry_1.default.maybeShowNotice();
+        (0, util_1.printBorder)();
+        if (!cmdObj.write) {
+            logger_1.default.info(`${chalk_1.default.green('✔')} Evaluation complete`);
+        }
+        else {
+            if (shareableUrl) {
+                logger_1.default.info(`${chalk_1.default.green('✔')} Evaluation complete: ${shareableUrl}`);
+            }
+            else {
+                logger_1.default.info(`${chalk_1.default.green('✔')} Evaluation complete.\n`);
+                logger_1.default.info(`» Run ${chalk_1.default.greenBright.bold('promptfoo view')} to use the local web viewer`);
+                logger_1.default.info(`» Run ${chalk_1.default.greenBright.bold('promptfoo share')} to create a shareable URL`);
+                logger_1.default.info(`» This project needs your feedback. What's one thing we can improve? ${chalk_1.default.greenBright.bold('https://forms.gle/YFLgTe1dKJKNSCsU7')}`);
+            }
+        }
+        (0, util_1.printBorder)();
+        logger_1.default.info(chalk_1.default.green.bold(`Successes: ${summary.stats.successes}`));
+        logger_1.default.info(chalk_1.default.red.bold(`Failures: ${summary.stats.failures}`));
+        logger_1.default.info(`Token usage: Total ${summary.stats.tokenUsage.total}, Prompt ${summary.stats.tokenUsage.prompt}, Completion ${summary.stats.tokenUsage.completion}, Cached ${summary.stats.tokenUsage.cached}`);
+        telemetry_1.default.record('command_used', {
+            name: 'eval',
+            watch: Boolean(cmdObj.watch),
+            duration: Math.round((Date.now() - startTime) / 1000),
+        });
+        await telemetry_1.default.send();
+        if (cmdObj.watch) {
+            if (initialization) {
+                const configPaths = (cmdObj.config || [defaultConfigPath]).filter(Boolean);
+                if (!configPaths.length) {
+                    logger_1.default.error('Could not locate config file(s) to watch');
+                    process.exit(1);
+                }
+                const basePath = path.dirname(configPaths[0]);
+                const promptPaths = Array.isArray(config.prompts)
+                    ? config.prompts
+                        .map((p) => {
+                        if (typeof p === 'string' && p.startsWith('file://')) {
+                            return path.resolve(basePath, p.slice('file://'.length));
+                        }
+                        else if (typeof p === 'object' && p.id && p.id.startsWith('file://')) {
+                            return path.resolve(basePath, p.id.slice('file://'.length));
+                        }
+                        return null;
+                    })
+                        .filter(Boolean)
+                    : [];
+                const providerPaths = Array.isArray(config.providers)
+                    ? config.providers
+                        .map((p) => typeof p === 'string' && p.startsWith('file://')
+                        ? path.resolve(basePath, p.slice('file://'.length))
+                        : null)
+                        .filter(Boolean)
+                    : [];
+                const varPaths = Array.isArray(config.tests)
+                    ? config.tests
+                        .flatMap((t) => {
+                        if (typeof t === 'string' && t.startsWith('file://')) {
+                            return path.resolve(basePath, t.slice('file://'.length));
+                        }
+                        else if (typeof t !== 'string' && t.vars) {
+                            return Object.values(t.vars).flatMap((v) => {
+                                if (typeof v === 'string' && v.startsWith('file://')) {
+                                    return path.resolve(basePath, v.slice('file://'.length));
+                                }
+                                return [];
+                            });
+                        }
+                        return [];
+                    })
+                        .filter(Boolean)
+                    : [];
+                const watchPaths = Array.from(new Set([...configPaths, ...promptPaths, ...providerPaths, ...varPaths]));
+                const watcher = chokidar_1.default.watch(watchPaths, { ignored: /^\./, persistent: true });
+                watcher
+                    .on('change', async (path) => {
+                    (0, util_1.printBorder)();
+                    logger_1.default.info(`File change detected: ${path}`);
+                    (0, util_1.printBorder)();
+                    await runEvaluation();
+                })
+                    .on('error', (error) => logger_1.default.error(`Watcher error: ${error}`))
+                    .on('ready', () => watchPaths.forEach((watchPath) => logger_1.default.info(`Watching for file changes on ${watchPath} ...`)));
+            }
+        }
+        else {
+            logger_1.default.info('Done.');
+            if (summary.stats.failures > 0) {
+                const exitCode = Number(process.env.PROMPTFOO_FAILED_TEST_EXIT_CODE);
+                process.exit(isNaN(exitCode) ? 100 : exitCode);
+            }
+        }
+    };
+    await runEvaluation(true /* initialization */);
+}
+function evalCommand(program, defaultConfig, defaultConfigPath, evaluateOptions) {
+    program
+        .command('eval')
+        .description('Evaluate prompts')
+        .option('-p, --prompts <paths...>', 'Paths to prompt files (.txt)')
+        .option('-r, --providers <name or path...>', 'One of: openai:chat, openai:completion, openai:<model name>, or path to custom API caller module')
+        .option('-c, --config <paths...>', 'Path to configuration file. Automatically loads promptfooconfig.js/json/yaml')
+        .option(
+    // TODO(ian): Remove `vars` for v1
+    '-v, --vars, -t, --tests <path>', 'Path to CSV with test cases', defaultConfig?.commandLineOptions?.vars)
+        .option('-a, --assertions <path>', 'Path to assertions file')
+        .option('--model-outputs <path>', 'Path to JSON containing list of LLM output strings')
+        .option('-t, --tests <path>', 'Path to CSV with test cases')
+        .option('-o, --output <paths...>', 'Path to output file (csv, txt, json, yaml, yml, html), default is no output file')
+        .option('-j, --max-concurrency <number>', 'Maximum number of concurrent API calls', defaultConfig.evaluateOptions?.maxConcurrency
+        ? String(defaultConfig.evaluateOptions.maxConcurrency)
+        : `${evaluator_1.DEFAULT_MAX_CONCURRENCY}`)
+        .option('--repeat <number>', 'Number of times to run each test', defaultConfig.evaluateOptions?.repeat ? String(defaultConfig.evaluateOptions.repeat) : '1')
+        .option('--delay <number>', 'Delay between each test (in milliseconds)', defaultConfig.evaluateOptions?.delay ? String(defaultConfig.evaluateOptions.delay) : '0')
+        .option('--table-cell-max-length <number>', 'Truncate console table cells to this length', '250')
+        .option('--suggest-prompts <number>', 'Generate N new prompts and append them to the prompt list')
+        .option('--prompt-prefix <path>', 'This prefix is prepended to every prompt', defaultConfig.defaultTest?.options?.prefix)
+        .option('--prompt-suffix <path>', 'This suffix is append to every prompt', defaultConfig.defaultTest?.options?.suffix)
+        .option('--no-write', 'Do not write results to promptfoo directory', defaultConfig?.commandLineOptions?.write)
+        .option('--no-cache', 'Do not read or write results to disk cache', 
+    // TODO(ian): Remove commandLineOptions.cache in v1
+    defaultConfig?.commandLineOptions?.cache ?? defaultConfig?.evaluateOptions?.cache)
+        .option('--no-progress-bar', 'Do not show progress bar')
+        .option('--table', 'Output table in CLI', defaultConfig?.commandLineOptions?.table ?? true)
+        .option('--no-table', 'Do not output table in CLI', defaultConfig?.commandLineOptions?.table)
+        .option('--share', 'Create a shareable URL', defaultConfig?.commandLineOptions?.share)
+        .option('--grader <provider>', 'Model that will grade outputs', defaultConfig?.commandLineOptions?.grader)
+        .option('--verbose', 'Show debug logs', defaultConfig?.commandLineOptions?.verbose)
+        .option('-w, --watch', 'Watch for changes in config and re-run')
+        .option('--env-file <path>', 'Path to .env file')
+        .option('--interactive-providers', 'Run providers interactively, one at a time', defaultConfig?.evaluateOptions?.interactiveProviders)
+        .option('-n, --filter-first-n <number>', 'Only run the first N tests')
+        .option('--filter-pattern <pattern>', 'Only run tests whose description matches the regular expression pattern')
+        .option('--filter-providers <providers>', 'Only run tests with these providers')
+        .option('--filter-failing <path>', 'Path to json output file')
+        .option('--var <key=value>', 'Set a variable in key=value format', (value, previous = {}) => {
+        const [key, val] = value.split('=');
+        if (!key || val === undefined) {
+            throw new Error('--var must be specified in key=value format.');
+        }
+        previous[key] = val;
+        return previous;
+    }, {})
+        .action((opts) => doEval(opts, defaultConfig, defaultConfigPath, evaluateOptions));
+}
+//# sourceMappingURL=eval.js.map

package/dist/src/commands/eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../../src/commands/eval.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyBA,wBAmOC;AAED,kCAoHC;AAlXD,kDAA0B;AAC1B,wDAAgC;AAEhC,2CAA6B;AAC7B,oCAAwC;AACxC,2DAAmC;AACnC,sCAA2C;AAC3C,4CAAiE;AACjE,oDAA6D;AAC7D,4CAA+C;AAC/C,oCAA8C;AAC9C,oCAAyC;AACzC,6DAAqC;AAErC,kCAOiB;AACjB,4DAAyD;AACzD,oDAAiD;AAE1C,KAAK,UAAU,MAAM,CAC1B,MAAoC,EACpC,aAAqC,EACrC,iBAAqC,EACrC,eAAgC;IAEhC,IAAA,eAAQ,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACzB,IAAI,MAAM,GAAuC,SAAS,CAAC;IAC3D,IAAI,SAAS,GAA0B,SAAS,CAAC;IACjD,IAAI,QAAQ,GAAuB,SAAS,CAAC;IAE7C,MAAM,aAAa,GAAG,KAAK,EAAE,cAAwB,EAAE,EAAE;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,gBAAgB;YACtB,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;SAC7B,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,gBAAgB;QAChB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,IAAA,oBAAW,EAAC,OAAO,CAAC,CAAC;QACvB,CAAC;QACD,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QACrE,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,gBAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAClC,IAAA,oBAAY,GAAE,CAAC;QACjB,CAAC;QAED,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,uBAAc,EAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;QAChF,kBAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAE7B,IAAI,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,cAAc,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QAC/D,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;QAE9C,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,cAAc,GAAG,CAAC,CAAC;YACnB,gBAAM,CAAC,IAAI,CACT,oCAAoC,KAAK,0CAA0C,CACpF,CAAC;QACJ,CAAC;QAED,SAAS,CAAC,KAAK,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE;YAC7C,MAAM,EAAE,MAAM,CAAC,YAAY;YAC3B,OAAO,EAAE,MAAM,CAAC,aAAa;YAC7B,OAAO,EAAE,MAAM,CAAC,aAAa;SAC9B,CAAC,CAAC;QAEH,SAAS,CAAC,SAAS,GAAG,IAAA,iCAAe,EAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,eAAe,CAAC,CAAC;QAEnF,MAAM,OAAO,GAAoB;YAC/B,eAAe,EAAE,IAAA,oBAAW,GAAE,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW;YACvE,cAAc,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS;YACzF,MAAM;YACN,KAAK,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;YACrD,oBAAoB,EAAE,MAAM,CAAC,oBAAoB;YACjD,GAAG,eAAe;SACnB,CAAC;QAEF,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,SAAS,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,IAAI,EAAE,CAAC;YACpD,SAAS,CAAC,WAAW,CAAC,OAAO,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,IAAI,EAAE,CAAC;YACpE,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,GAAG,MAAM,IAAA,2BAAe,EAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAChF,CAAC;QACD,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;YACf,SAAS,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,IAAI,EAAE,CAAC;YACpD,SAAS,CAAC,WAAW,CAAC,IAAI,GAAG,EAAE,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;QAChF,CAAC;QACD,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;YAC/B,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC;QACrC,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAA,oBAAQ,EAAC,SAAS,EAAE;YACxC,GAAG,OAAO;YACV,WAAW,EAAE,KAAK;SACnB,CAAC,CAAC;QAEH,MAAM,YAAY,GAChB,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAA,0BAAkB,EAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEpF,IAAI,MAAM,CAAC,KAAK,IAAI,IAAA,oBAAW,GAAE,KAAK,OAAO,EAAE,CAAC;YAC9C,mBAAmB;YACnB,MAAM,KAAK,GAAG,IAAA,qBAAa,EAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,kBAAkB,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;YAEpF,gBAAM,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YACrC,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;gBACnC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;gBAChD,gBAAM,CAAC,IAAI,CAAC,OAAO,QAAQ,YAAY,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,kBAAkB,CAAC,CAAC;YACtF,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,CAAC,KAAK,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YACxC,gBAAM,CAAC,KAAK,CACV,wIAAwI,IAAI,CAAC,SAAS,CACpJ,OAAO,CAAC,OAAO,CAChB,EAAE,CACJ,CAAC;QACJ,CAAC;QAED,MAAM,IAAA,6CAAsC,GAAE,CAAC;QAE/C,IAAI,MAAM,GAAkB,IAAI,CAAC;QACjC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACjB,MAAM,GAAG,MAAM,IAAA,6BAAsB,EAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;QAC9B,IAAI,UAAU,EAAE,CAAC;YACf,uBAAuB;YACvB,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;gBACnC,MAAM,IAAA,kBAAW,EAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;YACvE,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;gBACrC,MAAM,IAAA,2BAAoB,EAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;YAChF,CAAC;YACD,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,MAAM,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,mBAAS,CAAC,eAAe,EAAE,CAAC;QAE5B,IAAA,kBAAW,GAAE,CAAC;QACd,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAClB,gBAAM,CAAC,IAAI,CAAC,GAAG,eAAK,CAAC,KAAK,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;QACzD,CAAC;aAAM,CAAC;YACN,IAAI,YAAY,EAAE,CAAC;gBACjB,gBAAM,CAAC,IAAI,CAAC,GAAG,eAAK,CAAC,KAAK,CAAC,GAAG,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAC;YAC1E,CAAC;iBAAM,CAAC;gBACN,gBAAM,CAAC,IAAI,CAAC,GAAG,eAAK,CAAC,KAAK,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;gBAC1D,gBAAM,CAAC,IAAI,CACT,SAAS,eAAK,CAAC,WAAW,CAAC,IAAI,CAAC,gBAAgB,CAAC,8BAA8B,CAChF,CAAC;gBACF,gBAAM,CAAC,IAAI,CAAC,SAAS,eAAK,CAAC,WAAW,CAAC,IAAI,CAAC,iBAAiB,CAAC,4BAA4B,CAAC,CAAC;gBAC5F,gBAAM,CAAC,IAAI,CACT,wEAAwE,eAAK,CAAC,WAAW,CAAC,IAAI,CAC5F,qCAAqC,CACtC,EAAE,CACJ,CAAC;YACJ,CAAC;QACH,CAAC;QACD,IAAA,kBAAW,GAAE,CAAC;QACd,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACvE,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACnE,gBAAM,CAAC,IAAI,CACT,sBAAsB,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,YAAY,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,gBAAgB,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,YAAY,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAChM,CAAC;QAEF,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;YAC5B,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;SACtD,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACjB,IAAI,cAAc,EAAE,CAAC;gBACnB,MAAM,WAAW,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAa,CAAC;gBACvF,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;oBACxB,gBAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;oBACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAClB,CAAC;gBACD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC9C,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC;oBAC/C,CAAC,CAAE,MAAM,CAAC,OAAO;yBACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;wBACT,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;4BACrD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;wBAC3D,CAAC;6BAAM,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;4BACvE,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;wBAC9D,CAAC;wBACD,OAAO,IAAI,CAAC;oBACd,CAAC,CAAC;yBACD,MAAM,CAAC,OAAO,CAAc;oBACjC,CAAC,CAAC,EAAE,CAAC;gBACP,MAAM,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;oBACnD,CAAC,CAAE,MAAM,CAAC,SAAS;yBACd,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACT,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;wBAC9C,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;wBACnD,CAAC,CAAC,IAAI,CACT;yBACA,MAAM,CAAC,OAAO,CAAc;oBACjC,CAAC,CAAC,EAAE,CAAC;gBACP,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC1C,CAAC,CAAC,MAAM,CAAC,KAAK;yBACT,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;wBACb,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;4BACrD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;wBAC3D,CAAC;6BAAM,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;4BAC3C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;gCACzC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;oCACrD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;gCAC3D,CAAC;gCACD,OAAO,EAAE,CAAC;4BACZ,CAAC,CAAC,CAAC;wBACL,CAAC;wBACD,OAAO,EAAE,CAAC;oBACZ,CAAC,CAAC;yBACD,MAAM,CAAC,OAAO,CAAC;oBACpB,CAAC,CAAC,EAAE,CAAC;gBACP,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAC3B,IAAI,GAAG,CAAC,CAAC,GAAG,WAAW,EAAE,GAAG,WAAW,EAAE,GAAG,aAAa,EAAE,GAAG,QAAQ,CAAC,CAAC,CACzE,CAAC;gBACF,MAAM,OAAO,GAAG,kBAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;gBAEjF,OAAO;qBACJ,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;oBAC3B,IAAA,kBAAW,GAAE,CAAC;oBACd,gBAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;oBAC7C,IAAA,kBAAW,GAAE,CAAC;oBACd,MAAM,aAAa,EAAE,CAAC;gBACxB,CAAC,CAAC;qBACD,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAM,CAAC,KAAK,CAAC,kBAAkB,KAAK,EAAE,CAAC,CAAC;qBAC/D,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAChB,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,CAC/B,gBAAM,CAAC,IAAI,CAAC,gCAAgC,SAAS,MAAM,CAAC,CAC7D,CACF,CAAC;YACN,CAAC;QACH,CAAC;aAAM,CAAC;YACN,gBAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAErB,IAAI,OAAO,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;gBACrE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;AACjD,CAAC;AAED,SAAgB,WAAW,CACzB,OAAgB,EAChB,aAAqC,EACrC,iBAAqC,EACrC,eAAgC;IAEhC,OAAO;SACJ,OAAO,CAAC,MAAM,CAAC;SACf,WAAW,CAAC,kBAAkB,CAAC;SAC/B,MAAM,CAAC,0BAA0B,EAAE,8BAA8B,CAAC;SAClE,MAAM,CACL,mCAAmC,EACnC,kGAAkG,CACnG;SACA,MAAM,CACL,yBAAyB,EACzB,8EAA8E,CAC/E;SACA,MAAM;IACL,kCAAkC;IAClC,gCAAgC,EAChC,6BAA6B,EAC7B,aAAa,EAAE,kBAAkB,EAAE,IAAI,CACxC;SACA,MAAM,CAAC,yBAAyB,EAAE,yBAAyB,CAAC;SAC5D,MAAM,CAAC,wBAAwB,EAAE,oDAAoD,CAAC;SACtF,MAAM,CAAC,oBAAoB,EAAE,6BAA6B,CAAC;SAC3D,MAAM,CACL,yBAAyB,EACzB,kFAAkF,CACnF;SACA,MAAM,CACL,gCAAgC,EAChC,wCAAwC,EACxC,aAAa,CAAC,eAAe,EAAE,cAAc;QAC3C,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,cAAc,CAAC;QACtD,CAAC,CAAC,GAAG,mCAAuB,EAAE,CACjC;SACA,MAAM,CACL,mBAAmB,EACnB,kCAAkC,EAClC,aAAa,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAC3F;SACA,MAAM,CACL,kBAAkB,EAClB,2CAA2C,EAC3C,aAAa,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CACzF;SACA,MAAM,CACL,kCAAkC,EAClC,6CAA6C,EAC7C,KAAK,CACN;SACA,MAAM,CACL,4BAA4B,EAC5B,2DAA2D,CAC5D;SACA,MAAM,CACL,wBAAwB,EACxB,0CAA0C,EAC1C,aAAa,CAAC,WAAW,EAAE,OAAO,EAAE,MAAM,CAC3C;SACA,MAAM,CACL,wBAAwB,EACxB,uCAAuC,EACvC,aAAa,CAAC,WAAW,EAAE,OAAO,EAAE,MAAM,CAC3C;SACA,MAAM,CACL,YAAY,EACZ,6CAA6C,EAC7C,aAAa,EAAE,kBAAkB,EAAE,KAAK,CACzC;SACA,MAAM,CACL,YAAY,EACZ,4CAA4C;IAC5C,mDAAmD;IACnD,aAAa,EAAE,kBAAkB,EAAE,KAAK,IAAI,aAAa,EAAE,eAAe,EAAE,KAAK,CAClF;SACA,MAAM,CAAC,mBAAmB,EAAE,0BAA0B,CAAC;SACvD,MAAM,CAAC,SAAS,EAAE,qBAAqB,EAAE,aAAa,EAAE,kBAAkB,EAAE,KAAK,IAAI,IAAI,CAAC;SAC1F,MAAM,CAAC,YAAY,EAAE,4BAA4B,EAAE,aAAa,EAAE,kBAAkB,EAAE,KAAK,CAAC;SAC5F,MAAM,CAAC,SAAS,EAAE,wBAAwB,EAAE,aAAa,EAAE,kBAAkB,EAAE,KAAK,CAAC;SACrF,MAAM,CACL,qBAAqB,EACrB,+BAA+B,EAC/B,aAAa,EAAE,kBAAkB,EAAE,MAAM,CAC1C;SACA,MAAM,CAAC,WAAW,EAAE,iBAAiB,EAAE,aAAa,EAAE,kBAAkB,EAAE,OAAO,CAAC;SAClF,MAAM,CAAC,aAAa,EAAE,wCAAwC,CAAC;SAC/D,MAAM,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;SAChD,MAAM,CACL,yBAAyB,EACzB,4CAA4C,EAC5C,aAAa,EAAE,eAAe,EAAE,oBAAoB,CACrD;SACA,MAAM,CAAC,+BAA+B,EAAE,4BAA4B,CAAC;SACrE,MAAM,CACL,4BAA4B,EAC5B,yEAAyE,CAC1E;SACA,MAAM,CAAC,gCAAgC,EAAE,qCAAqC,CAAC;SAC/E,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,CAAC;SAC7D,MAAM,CACL,mBAAmB,EACnB,oCAAoC,EACpC,CAAC,KAAK,EAAE,WAAmC,EAAE,EAAE,EAAE;QAC/C,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,GAAG,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;QACpB,OAAO,QAAQ,CAAC;IAClB,CAAC,EACD,EAAE,CACH;SACA,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,aAAa,EAAE,iBAAiB,EAAE,eAAe,CAAC,CAAC,CAAC;AACvF,CAAC"}

package/dist/src/commands/generate.d.ts

@@ -0,0 +1,20 @@
+import { Command } from 'commander';
+import { UnifiedConfig } from '../types';
+interface RedteamGenerateOptions {
+    addPlugins?: string[];
+    cache: boolean;
+    config?: string;
+    envFile?: string;
+    injectVar?: string;
+    output?: string;
+    plugins?: string[];
+    provider?: string;
+    purpose?: string;
+    write: boolean;
+    defaultConfig: Partial<UnifiedConfig>;
+    defaultConfigPath: string | undefined;
+}
+export declare function doGenerateRedteam({ addPlugins, cache, config, envFile, injectVar, output, plugins, provider, purpose, write, defaultConfig, defaultConfigPath, }: RedteamGenerateOptions): Promise<void>;
+export declare function generateCommand(program: Command, defaultConfig: Partial<UnifiedConfig>, defaultConfigPath: string | undefined): void;
+export {};
+//# sourceMappingURL=generate.d.ts.map

package/dist/src/commands/generate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../src/commands/generate.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAepC,OAAO,EAAa,aAAa,EAAE,MAAM,UAAU,CAAC;AAGpD,UAAU,sBAAsB;IAE9B,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;IAGf,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACtC,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;CACvC;AAED,wBAAsB,iBAAiB,CAAC,EACtC,UAAU,EACV,KAAK,EACL,MAAM,EACN,OAAO,EACP,SAAS,EACT,MAAM,EACN,OAAO,EACP,QAAQ,EACR,OAAO,EACP,KAAK,EACL,aAAa,EACb,iBAAiB,GAClB,EAAE,sBAAsB,iBAuFxB;AAED,wBAAgB,eAAe,CAC7B,OAAO,EAAE,OAAO,EAChB,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,EACrC,iBAAiB,EAAE,MAAM,GAAG,SAAS,QA6ItC"}

package/dist/src/commands/generate.js

@@ -0,0 +1,215 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.doGenerateRedteam = doGenerateRedteam;
+exports.generateCommand = generateCommand;
+const chalk_1 = __importDefault(require("chalk"));
+const dedent_1 = __importDefault(require("dedent"));
+const fs = __importStar(require("fs"));
+const js_yaml_1 = __importDefault(require("js-yaml"));
+const cache_1 = require("../cache");
+const config_1 = require("../config");
+const logger_1 = __importDefault(require("../logger"));
+const redteam_1 = require("../redteam");
+const constants_1 = require("../redteam/constants");
+const telemetry_1 = __importDefault(require("../telemetry"));
+const testCases_1 = require("../testCases");
+const util_1 = require("../util");
+async function doGenerateRedteam({ addPlugins, cache, config, envFile, injectVar, output, plugins, provider, purpose, write, defaultConfig, defaultConfigPath, }) {
+    (0, util_1.setupEnv)(envFile);
+    if (!cache) {
+        logger_1.default.info('Cache is disabled.');
+        (0, cache_1.disableCache)();
+    }
+    let testSuite;
+    const configPath = config || defaultConfigPath;
+    if (configPath) {
+        const resolved = await (0, config_1.resolveConfigs)({
+            config: [configPath],
+        }, defaultConfig);
+        testSuite = resolved.testSuite;
+    }
+    else if (purpose) {
+        // There is a purpose, so we can just have a dummy testsuite for standalone invocation
+        testSuite = {
+            prompts: [],
+            providers: [],
+            tests: [],
+        };
+    }
+    else {
+        logger_1.default.info(chalk_1.default.red(`\nCan't generate without configuration - run ${chalk_1.default.yellow.bold('promptfoo redteam init')} first`));
+        return;
+    }
+    const startTime = Date.now();
+    telemetry_1.default.record('command_used', {
+        name: 'generate redteam - started',
+        numPrompts: testSuite.prompts.length,
+        numTestsExisting: (testSuite.tests || []).length,
+    });
+    await telemetry_1.default.send();
+    const redteamTests = await (0, redteam_1.synthesizeFromTestSuite)(testSuite, {
+        purpose,
+        injectVar,
+        plugins: addPlugins && addPlugins.length > 0
+            ? Array.from(plugins || constants_1.DEFAULT_PLUGINS).concat(addPlugins)
+            : plugins,
+        provider,
+    });
+    if (output) {
+        const existingYaml = configPath
+            ? js_yaml_1.default.load(fs.readFileSync(configPath, 'utf8'))
+            : {};
+        const updatedYaml = {
+            ...existingYaml,
+            tests: redteamTests,
+            metadata: {
+                ...existingYaml.metadata,
+                redteam: true,
+            },
+        };
+        fs.writeFileSync(output, js_yaml_1.default.dump(updatedYaml, { skipInvalid: true }));
+        (0, util_1.printBorder)();
+        logger_1.default.info(`Wrote ${redteamTests.length} new test cases to ${output}`);
+        (0, util_1.printBorder)();
+    }
+    else if (write && configPath) {
+        const existingConfig = js_yaml_1.default.load(fs.readFileSync(configPath, 'utf8'));
+        existingConfig.tests = [...(existingConfig.tests || []), ...redteamTests];
+        fs.writeFileSync(configPath, js_yaml_1.default.dump(existingConfig));
+        logger_1.default.info(`\nWrote ${redteamTests.length} new test cases to ${configPath}`);
+        logger_1.default.info('\n' + chalk_1.default.green(`Run ${chalk_1.default.bold('promptfoo eval')} to run the generated tests`));
+    }
+    else {
+        logger_1.default.info(js_yaml_1.default.dump(redteamTests, { skipInvalid: true }));
+    }
+    telemetry_1.default.record('command_used', {
+        name: 'generate redteam',
+        numPrompts: testSuite.prompts.length,
+        numTestsExisting: (testSuite.tests || []).length,
+        numTestsGenerated: redteamTests.length,
+        duration: Math.round((Date.now() - startTime) / 1000),
+    });
+    await telemetry_1.default.send();
+}
+function generateCommand(program, defaultConfig, defaultConfigPath) {
+    const generateCommand = program.command('generate').description('Generate synthetic data');
+    generateCommand
+        .command('dataset')
+        .description('Generate test cases')
+        .option('-i, --instructions [instructions]', 'Additional instructions to follow while generating test cases')
+        .option('-c, --config [path]', 'Path to configuration file. Defaults to promptfooconfig.yaml')
+        .option('-o, --output [path]', 'Path to output file')
+        .option('-w, --write', 'Write results to promptfoo configuration file')
+        .option('--numPersonas <number>', 'Number of personas to generate', '5')
+        .option('--numTestCasesPerPersona <number>', 'Number of test cases per persona', '3')
+        .option('--no-cache', 'Do not read or write results to disk cache', false)
+        .option('--env-file <path>', 'Path to .env file')
+        .action(async (options) => {
+        (0, util_1.setupEnv)(options.envFile);
+        if (!options.cache) {
+            logger_1.default.info('Cache is disabled.');
+            (0, cache_1.disableCache)();
+        }
+        let testSuite;
+        const configPath = options.config || defaultConfigPath;
+        if (configPath) {
+            const resolved = await (0, config_1.resolveConfigs)({
+                config: [configPath],
+            }, defaultConfig);
+            testSuite = resolved.testSuite;
+        }
+        else {
+            throw new Error('Could not find config file. Please use `--config`');
+        }
+        const startTime = Date.now();
+        telemetry_1.default.record('command_used', {
+            name: 'generate_dataset - started',
+            numPrompts: testSuite.prompts.length,
+            numTestsExisting: (testSuite.tests || []).length,
+        });
+        await telemetry_1.default.send();
+        const results = await (0, testCases_1.synthesizeFromTestSuite)(testSuite, {
+            instructions: options.instructions,
+            numPersonas: parseInt(options.numPersonas, 10),
+            numTestCasesPerPersona: parseInt(options.numTestCasesPerPersona, 10),
+        });
+        const configAddition = { tests: results.map((result) => ({ vars: result })) };
+        const yamlString = js_yaml_1.default.dump(configAddition);
+        if (options.output) {
+            fs.writeFileSync(options.output, yamlString);
+            (0, util_1.printBorder)();
+            logger_1.default.info(`Wrote ${results.length} new test cases to ${options.output}`);
+            (0, util_1.printBorder)();
+        }
+        else {
+            (0, util_1.printBorder)();
+            logger_1.default.info('New test Cases');
+            (0, util_1.printBorder)();
+            logger_1.default.info(yamlString);
+        }
+        (0, util_1.printBorder)();
+        if (options.write && configPath) {
+            const existingConfig = js_yaml_1.default.load(fs.readFileSync(configPath, 'utf8'));
+            existingConfig.tests = [...(existingConfig.tests || []), ...configAddition.tests];
+            fs.writeFileSync(configPath, js_yaml_1.default.dump(existingConfig));
+            logger_1.default.info(`Wrote ${results.length} new test cases to ${configPath}`);
+            logger_1.default.info(chalk_1.default.green(`Run ${chalk_1.default.bold('promptfoo eval')} to run the generated tests`));
+        }
+        else {
+            logger_1.default.info(`Copy the above test cases or run ${chalk_1.default.greenBright('promptfoo generate dataset --write')} to write directly to the config`);
+        }
+        telemetry_1.default.record('command_used', {
+            name: 'generate_dataset',
+            numPrompts: testSuite.prompts.length,
+            numTestsExisting: (testSuite.tests || []).length,
+            numTestsGenerated: results.length,
+            duration: Math.round((Date.now() - startTime) / 1000),
+        });
+        await telemetry_1.default.send();
+    });
+    generateCommand
+        .command('redteam')
+        .description('Generate adversarial test cases')
+        .option('-c, --config [path]', 'Path to configuration file. Defaults to promptfooconfig.yaml')
+        .option('-o, --output [path]', 'Path to output file')
+        .option('-w, --write', 'Write results to promptfoo configuration file')
+        .option('--purpose <purpose>', 'Set the system purpose. If not set, the system purpose will be inferred from the config file')
+        .option('--provider <provider>', `Provider to use for generating adversarial tests. Defaults to: ${constants_1.REDTEAM_MODEL}`)
+        .option('--injectVar <varname>', 'Override the variable to inject user input into the prompt. If not set, the variable will default to {{query}}')
+        .option('--plugins <plugins>', (0, dedent_1.default) `Comma-separated list of plugins to use. Defaults to:
+        \n- ${Array.from(constants_1.DEFAULT_PLUGINS).sort().join('\n- ')}\n\n
+    `, (val) => val.split(',').map((x) => x.trim()))
+        .option('--add-plugins <plugins>', (0, dedent_1.default) `Comma-separated list of plugins to run in addition to the default plugins:
+        \n- ${constants_1.ADDITIONAL_PLUGINS.sort().join('\n- ')}\n\n
+      `, (val) => val.split(',').map((x) => x.trim()))
+        .option('--no-cache', 'Do not read or write results to disk cache', false)
+        .option('--env-file <path>', 'Path to .env file')
+        .action((opts) => doGenerateRedteam({ ...opts, defaultConfig, defaultConfigPath }));
+}
+//# sourceMappingURL=generate.js.map

package/dist/src/commands/generate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../../src/commands/generate.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqCA,8CAoGC;AAED,0CAgJC;AA3RD,kDAA0B;AAE1B,oDAA4B;AAC5B,uCAAyB;AACzB,sDAA2B;AAC3B,oCAAwC;AACxC,sCAA2C;AAC3C,uDAA+B;AAC/B,wCAAuF;AACvF,oDAI8B;AAC9B,6DAAqC;AACrC,4CAAuD;AAEvD,kCAAgD;AAoBzC,KAAK,UAAU,iBAAiB,CAAC,EACtC,UAAU,EACV,KAAK,EACL,MAAM,EACN,OAAO,EACP,SAAS,EACT,MAAM,EACN,OAAO,EACP,QAAQ,EACR,OAAO,EACP,KAAK,EACL,aAAa,EACb,iBAAiB,GACM;IACvB,IAAA,eAAQ,EAAC,OAAO,CAAC,CAAC;IAClB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,gBAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAClC,IAAA,oBAAY,GAAE,CAAC;IACjB,CAAC;IAED,IAAI,SAAoB,CAAC;IACzB,MAAM,UAAU,GAAG,MAAM,IAAI,iBAAiB,CAAC;IAC/C,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,QAAQ,GAAG,MAAM,IAAA,uBAAc,EACnC;YACE,MAAM,EAAE,CAAC,UAAU,CAAC;SACrB,EACD,aAAa,CACd,CAAC;QACF,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC;IACjC,CAAC;SAAM,IAAI,OAAO,EAAE,CAAC;QACnB,sFAAsF;QACtF,SAAS,GAAG;YACV,OAAO,EAAE,EAAE;YACX,SAAS,EAAE,EAAE;YACb,KAAK,EAAE,EAAE;SACV,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,GAAG,CACP,gDAAgD,eAAK,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CACpG,CACF,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QAC/B,IAAI,EAAE,4BAA4B;QAClC,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM;QACpC,gBAAgB,EAAE,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM;KACjD,CAAC,CAAC;IACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;IAEvB,MAAM,YAAY,GAAG,MAAM,IAAA,iCAA8B,EAAC,SAAS,EAAE;QACnE,OAAO;QACP,SAAS;QACT,OAAO,EACL,UAAU,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;YACjC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,2BAAuB,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC;YACnE,CAAC,CAAC,OAAO;QACb,QAAQ;KACT,CAAC,CAAC;IAEH,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,YAAY,GAAG,UAAU;YAC7B,CAAC,CAAE,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAA4B;YAC5E,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,WAAW,GAAG;YAClB,GAAG,YAAY;YACf,KAAK,EAAE,YAAY;YACnB,QAAQ,EAAE;gBACR,GAAG,YAAY,CAAC,QAAQ;gBACxB,OAAO,EAAE,IAAI;aACd;SACF,CAAC;QACF,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE,iBAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACxE,IAAA,kBAAW,GAAE,CAAC;QACd,gBAAM,CAAC,IAAI,CAAC,SAAS,YAAY,CAAC,MAAM,sBAAsB,MAAM,EAAE,CAAC,CAAC;QACxE,IAAA,kBAAW,GAAE,CAAC;IAChB,CAAC;SAAM,IAAI,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,cAAc,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAA2B,CAAC;QAChG,cAAc,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,GAAG,YAAY,CAAC,CAAC;QAC1E,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,iBAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;QACxD,gBAAM,CAAC,IAAI,CAAC,WAAW,YAAY,CAAC,MAAM,sBAAsB,UAAU,EAAE,CAAC,CAAC;QAC9E,gBAAM,CAAC,IAAI,CACT,IAAI,GAAG,eAAK,CAAC,KAAK,CAAC,OAAO,eAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,6BAA6B,CAAC,CACrF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,gBAAM,CAAC,IAAI,CAAC,iBAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QAC/B,IAAI,EAAE,kBAAkB;QACxB,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM;QACpC,gBAAgB,EAAE,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM;QAChD,iBAAiB,EAAE,YAAY,CAAC,MAAM;QACtC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;KACtD,CAAC,CAAC;IACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAED,SAAgB,eAAe,CAC7B,OAAgB,EAChB,aAAqC,EACrC,iBAAqC;IAErC,MAAM,eAAe,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,WAAW,CAAC,yBAAyB,CAAC,CAAC;IAE3F,eAAe;SACZ,OAAO,CAAC,SAAS,CAAC;SAClB,WAAW,CAAC,qBAAqB,CAAC;SAClC,MAAM,CACL,mCAAmC,EACnC,+DAA+D,CAChE;SACA,MAAM,CAAC,qBAAqB,EAAE,8DAA8D,CAAC;SAC7F,MAAM,CAAC,qBAAqB,EAAE,qBAAqB,CAAC;SACpD,MAAM,CAAC,aAAa,EAAE,+CAA+C,CAAC;SACtE,MAAM,CAAC,wBAAwB,EAAE,gCAAgC,EAAE,GAAG,CAAC;SACvE,MAAM,CAAC,mCAAmC,EAAE,kCAAkC,EAAE,GAAG,CAAC;SACpF,MAAM,CAAC,YAAY,EAAE,4CAA4C,EAAE,KAAK,CAAC;SACzE,MAAM,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;SAChD,MAAM,CACL,KAAK,EAAE,OASN,EAAE,EAAE;QACH,IAAA,eAAQ,EAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC1B,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,gBAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAClC,IAAA,oBAAY,GAAE,CAAC;QACjB,CAAC;QAED,IAAI,SAAoB,CAAC;QACzB,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,IAAI,iBAAiB,CAAC;QACvD,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,MAAM,IAAA,uBAAc,EACnC;gBACE,MAAM,EAAE,CAAC,UAAU,CAAC;aACrB,EACD,aAAa,CACd,CAAC;YACF,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QACvE,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,4BAA4B;YAClC,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM;YACpC,gBAAgB,EAAE,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM;SACjD,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,MAAM,OAAO,GAAG,MAAM,IAAA,mCAAuB,EAAC,SAAS,EAAE;YACvD,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;YAC9C,sBAAsB,EAAE,QAAQ,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC;SACrE,CAAC,CAAC;QACH,MAAM,cAAc,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC;QAC9E,MAAM,UAAU,GAAG,iBAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC7C,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;YAC7C,IAAA,kBAAW,GAAE,CAAC;YACd,gBAAM,CAAC,IAAI,CAAC,SAAS,OAAO,CAAC,MAAM,sBAAsB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAC3E,IAAA,kBAAW,GAAE,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,IAAA,kBAAW,GAAE,CAAC;YACd,gBAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAC9B,IAAA,kBAAW,GAAE,CAAC;YACd,gBAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC1B,CAAC;QAED,IAAA,kBAAW,GAAE,CAAC;QACd,IAAI,OAAO,CAAC,KAAK,IAAI,UAAU,EAAE,CAAC;YAChC,MAAM,cAAc,GAAG,iBAAI,CAAC,IAAI,CAC9B,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CACV,CAAC;YAC5B,cAAc,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YAClF,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,iBAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;YACxD,gBAAM,CAAC,IAAI,CAAC,SAAS,OAAO,CAAC,MAAM,sBAAsB,UAAU,EAAE,CAAC,CAAC;YACvE,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,KAAK,CAAC,OAAO,eAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,6BAA6B,CAAC,CAC9E,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,gBAAM,CAAC,IAAI,CACT,oCAAoC,eAAK,CAAC,WAAW,CACnD,oCAAoC,CACrC,kCAAkC,CACpC,CAAC;QACJ,CAAC;QAED,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,kBAAkB;YACxB,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM;YACpC,gBAAgB,EAAE,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM;YAChD,iBAAiB,EAAE,OAAO,CAAC,MAAM;YACjC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;SACtD,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC,CACF,CAAC;IAEJ,eAAe;SACZ,OAAO,CAAC,SAAS,CAAC;SAClB,WAAW,CAAC,iCAAiC,CAAC;SAC9C,MAAM,CAAC,qBAAqB,EAAE,8DAA8D,CAAC;SAC7F,MAAM,CAAC,qBAAqB,EAAE,qBAAqB,CAAC;SACpD,MAAM,CAAC,aAAa,EAAE,+CAA+C,CAAC;SACtE,MAAM,CACL,qBAAqB,EACrB,8FAA8F,CAC/F;SACA,MAAM,CACL,uBAAuB,EACvB,kEAAkE,yBAAa,EAAE,CAClF;SACA,MAAM,CACL,uBAAuB,EACvB,gHAAgH,CACjH;SACA,MAAM,CACL,qBAAqB,EACrB,IAAA,gBAAM,EAAA;cACE,KAAK,CAAC,IAAI,CAAC,2BAAuB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;KAChE,EACC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAC7C;SACA,MAAM,CACL,yBAAyB,EACzB,IAAA,gBAAM,EAAA;cACE,8BAA0B,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;OACrD,EACD,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAC7C;SACA,MAAM,CAAC,YAAY,EAAE,4CAA4C,EAAE,KAAK,CAAC;SACzE,MAAM,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;SAChD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,GAAG,IAAI,EAAE,aAAa,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC;AACxF,CAAC"}

package/dist/src/commands/redteam.d.ts

@@ -0,0 +1,10 @@
+import { Command } from 'commander';
+interface RunRedteamOptions {
+    config: string;
+    cache: boolean;
+    envFile: string;
+}
+export declare function doRunRedteam(cmdObj: RunRedteamOptions): Promise<void>;
+export declare function redteamCommand(program: Command): void;
+export {};
+//# sourceMappingURL=redteam.d.ts.map

package/dist/src/commands/redteam.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/commands/redteam.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAUpC,UAAU,iBAAiB;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,wBAAsB,YAAY,CAAC,MAAM,EAAE,iBAAiB,iBAAI;AAEhE,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,QA0K9C"}