promptfoo 0.69.2 → 0.70.1

package/dist/src/main.js

@@ -1,207 +1,35 @@
 #!/usr/bin/env node
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const chalk_1 = __importDefault(require("chalk"));
-const chokidar_1 = __importDefault(require("chokidar"));
 const commander_1 = require("commander");
-const dedent_1 = __importDefault(require("dedent"));
 const fs_1 = __importDefault(require("fs"));
-const js_yaml_1 = __importDefault(require("js-yaml"));
 const path_1 = __importDefault(require("path"));
 const readline_1 = __importDefault(require("readline"));
-const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
-const assertions_1 = require("./assertions");
-const validateAssertions_1 = require("./assertions/validateAssertions");
 const cache_1 = require("./cache");
 const checkNodeVersion_1 = require("./checkNodeVersion");
-const cliState_1 = __importDefault(require("./cliState"));
 const config_1 = require("./commands/config");
 const delete_1 = require("./commands/delete");
-const filterProviders_1 = require("./commands/eval/filterProviders");
-const filterTests_1 = require("./commands/eval/filterTests");
+const eval_1 = require("./commands/eval");
 const export_1 = require("./commands/export");
+const generate_1 = require("./commands/generate");
 const import_1 = require("./commands/import");
 const list_1 = require("./commands/list");
+const redteam_1 = require("./commands/redteam");
 const show_1 = require("./commands/show");
+const config_2 = require("./config");
 const esm_1 = require("./esm");
-const evaluator_1 = require("./evaluator");
 const feedback_1 = require("./feedback");
-const logger_1 = __importStar(require("./logger"));
+const logger_1 = __importDefault(require("./logger"));
 const onboarding_1 = require("./onboarding");
-const prompts_1 = require("./prompts");
-const providers_1 = require("./providers");
-const redteam_1 = require("./redteam");
-const constants_1 = require("./redteam/constants");
 const share_1 = require("./share");
-const table_1 = require("./table");
 const telemetry_1 = __importDefault(require("./telemetry"));
-const testCases_1 = require("./testCases");
-const types_1 = require("./types");
 const updates_1 = require("./updates");
 const util_1 = require("./util");
 const server_1 = require("./web/server");
-async function resolveConfigs(cmdObj, defaultConfig) {
-    // Config parsing
-    let fileConfig = {};
-    const configPaths = cmdObj.config;
-    if (configPaths) {
-        fileConfig = await (0, util_1.readConfigs)(configPaths);
-    }
-    // Standalone assertion mode
-    if (cmdObj.assertions) {
-        if (!cmdObj.modelOutputs) {
-            logger_1.default.error(chalk_1.default.red('You must provide --model-outputs when using --assertions'));
-            process.exit(1);
-        }
-        const modelOutputs = JSON.parse(fs_1.default.readFileSync(path_1.default.join(process.cwd(), cmdObj.modelOutputs), 'utf8'));
-        const assertions = await (0, assertions_1.readAssertions)(cmdObj.assertions);
-        fileConfig.prompts = ['{{output}}'];
-        fileConfig.providers = ['echo'];
-        fileConfig.tests = modelOutputs.map((output) => {
-            if (typeof output === 'string') {
-                return {
-                    vars: {
-                        output,
-                    },
-                    assert: assertions,
-                };
-            }
-            return {
-                vars: {
-                    output: output.output,
-                    ...(output.tags === undefined ? {} : { tags: output.tags.join(', ') }),
-                },
-                assert: assertions,
-            };
-        });
-    }
-    // Use basepath in cases where path was supplied in the config file
-    const basePath = configPaths ? path_1.default.dirname(configPaths[0]) : '';
-    const defaultTestRaw = fileConfig.defaultTest || defaultConfig.defaultTest;
-    const config = {
-        description: fileConfig.description || defaultConfig.description,
-        prompts: cmdObj.prompts || fileConfig.prompts || defaultConfig.prompts || [],
-        providers: cmdObj.providers || fileConfig.providers || defaultConfig.providers || [],
-        tests: cmdObj.tests || cmdObj.vars || fileConfig.tests || defaultConfig.tests || [],
-        scenarios: fileConfig.scenarios || defaultConfig.scenarios,
-        env: fileConfig.env || defaultConfig.env,
-        sharing: process.env.PROMPTFOO_DISABLE_SHARING === '1'
-            ? false
-            : fileConfig.sharing ?? defaultConfig.sharing ?? true,
-        defaultTest: defaultTestRaw ? await (0, testCases_1.readTest)(defaultTestRaw, basePath) : undefined,
-        derivedMetrics: fileConfig.derivedMetrics || defaultConfig.derivedMetrics,
-        outputPath: cmdObj.output || fileConfig.outputPath || defaultConfig.outputPath,
-        metadata: fileConfig.metadata || defaultConfig.metadata,
-    };
-    // Validation
-    if (!config.prompts || config.prompts.length === 0) {
-        logger_1.default.error(chalk_1.default.red('You must provide at least 1 prompt'));
-        process.exit(1);
-    }
-    if (!config.providers || config.providers.length === 0) {
-        logger_1.default.error(chalk_1.default.red('You must specify at least 1 provider (for example, openai:gpt-4o)'));
-        process.exit(1);
-    }
-    (0, tiny_invariant_1.default)(Array.isArray(config.providers), 'providers must be an array');
-    config.providers.forEach((provider) => {
-        const result = types_1.ProviderSchema.safeParse(provider);
-        if (!result.success) {
-            const errors = result.error.errors
-                .map((err) => {
-                return `- ${err.message}`;
-            })
-                .join('\n');
-            const providerString = typeof provider === 'string' ? provider : JSON.stringify(provider);
-            logger_1.default.warn(chalk_1.default.yellow((0, dedent_1.default) `
-              Provider: ${providerString} encountered errors during schema validation:
-
-                ${errors}
-
-              Please double check your configuration.` + '\n'));
-        }
-    });
-    // Parse prompts, providers, and tests
-    const parsedPrompts = await (0, prompts_1.readPrompts)(config.prompts, cmdObj.prompts ? undefined : basePath);
-    const parsedProviders = await (0, providers_1.loadApiProviders)(config.providers, {
-        env: config.env,
-        basePath,
-    });
-    const parsedTests = await (0, testCases_1.readTests)(config.tests || [], cmdObj.tests ? undefined : basePath);
-    // Parse testCases for each scenario
-    if (fileConfig.scenarios) {
-        for (const scenario of fileConfig.scenarios) {
-            const parsedScenarioTests = await (0, testCases_1.readTests)(scenario.tests, cmdObj.tests ? undefined : basePath);
-            scenario.tests = parsedScenarioTests;
-            const filteredTests = await (0, filterTests_1.filterTests)({
-                ...scenario,
-                providers: parsedProviders,
-                prompts: parsedPrompts,
-            }, {
-                firstN: cmdObj.filterFirstN,
-                pattern: cmdObj.filterPattern,
-                failing: cmdObj.filterFailing,
-            });
-            (0, tiny_invariant_1.default)(filteredTests, 'filteredTests are undefined');
-            scenario.tests = filteredTests;
-        }
-    }
-    const parsedProviderPromptMap = (0, prompts_1.readProviderPromptMap)(config, parsedPrompts);
-    if (parsedPrompts.length === 0) {
-        logger_1.default.error(chalk_1.default.red('No prompts found'));
-        process.exit(1);
-    }
-    const defaultTest = {
-        options: {
-            prefix: cmdObj.promptPrefix,
-            suffix: cmdObj.promptSuffix,
-            provider: cmdObj.grader,
-            // rubricPrompt
-            ...(config.defaultTest?.options || {}),
-        },
-        ...config.defaultTest,
-    };
-    const testSuite = {
-        description: config.description,
-        prompts: parsedPrompts,
-        providers: parsedProviders,
-        providerPromptMap: parsedProviderPromptMap,
-        tests: parsedTests,
-        scenarios: config.scenarios,
-        defaultTest,
-        derivedMetrics: config.derivedMetrics,
-        nunjucksFilters: await (0, util_1.readFilters)(fileConfig.nunjucksFilters || defaultConfig.nunjucksFilters || {}),
-    };
-    if (testSuite.tests) {
-        (0, validateAssertions_1.validateAssertions)(testSuite.tests);
-    }
-    return { config, testSuite, basePath };
-}
 async function main() {
     await (0, updates_1.checkForUpdates)();
     const pwd = process.cwd();
@@ -214,7 +42,7 @@ async function main() {
     let defaultConfig = {};
     let defaultConfigPath;
     for (const _path of potentialPaths) {
-        const maybeConfig = await (0, util_1.maybeReadConfig)(_path);
+        const maybeConfig = await (0, config_2.maybeReadConfig)(_path);
         if (maybeConfig) {
             defaultConfig = maybeConfig;
             defaultConfigPath = _path;
@@ -339,399 +167,15 @@ async function main() {
         .action((message) => {
         (0, feedback_1.gatherFeedback)(message);
     });
-    const generateCommand = program.command('generate').description('Generate synthetic data');
-    generateCommand
-        .command('dataset')
-        .description('Generate test cases')
-        .option('-i, --instructions [instructions]', 'Additional instructions to follow while generating test cases')
-        .option('-c, --config [path]', 'Path to configuration file. Defaults to promptfooconfig.yaml')
-        .option('-o, --output [path]', 'Path to output file')
-        .option('-w, --write', 'Write results to promptfoo configuration file')
-        .option('--numPersonas <number>', 'Number of personas to generate', '5')
-        .option('--numTestCasesPerPersona <number>', 'Number of test cases per persona', '3')
-        .option('--no-cache', 'Do not read or write results to disk cache', false)
-        .option('--env-file <path>', 'Path to .env file')
-        .action(async (options) => {
-        (0, util_1.setupEnv)(options.envFile);
-        if (!options.cache) {
-            logger_1.default.info('Cache is disabled.');
-            (0, cache_1.disableCache)();
-        }
-        let testSuite;
-        const configPath = options.config || defaultConfigPath;
-        if (configPath) {
-            const resolved = await resolveConfigs({
-                config: [configPath],
-            }, defaultConfig);
-            testSuite = resolved.testSuite;
-        }
-        else {
-            throw new Error('Could not find config file. Please use `--config`');
-        }
-        const startTime = Date.now();
-        telemetry_1.default.record('command_used', {
-            name: 'generate_dataset - started',
-            numPrompts: testSuite.prompts.length,
-            numTestsExisting: (testSuite.tests || []).length,
-        });
-        await telemetry_1.default.send();
-        const results = await (0, testCases_1.synthesizeFromTestSuite)(testSuite, {
-            instructions: options.instructions,
-            numPersonas: parseInt(options.numPersonas, 10),
-            numTestCasesPerPersona: parseInt(options.numTestCasesPerPersona, 10),
-        });
-        const configAddition = { tests: results.map((result) => ({ vars: result })) };
-        const yamlString = js_yaml_1.default.dump(configAddition);
-        if (options.output) {
-            fs_1.default.writeFileSync(options.output, yamlString);
-            (0, util_1.printBorder)();
-            logger_1.default.info(`Wrote ${results.length} new test cases to ${options.output}`);
-            (0, util_1.printBorder)();
-        }
-        else {
-            (0, util_1.printBorder)();
-            logger_1.default.info('New test Cases');
-            (0, util_1.printBorder)();
-            logger_1.default.info(yamlString);
-        }
-        (0, util_1.printBorder)();
-        if (options.write && configPath) {
-            const existingConfig = js_yaml_1.default.load(fs_1.default.readFileSync(configPath, 'utf8'));
-            existingConfig.tests = [...(existingConfig.tests || []), ...configAddition.tests];
-            fs_1.default.writeFileSync(configPath, js_yaml_1.default.dump(existingConfig));
-            logger_1.default.info(`Wrote ${results.length} new test cases to ${configPath}`);
-        }
-        else {
-            logger_1.default.info(`Copy the above test cases or run ${chalk_1.default.greenBright('promptfoo generate dataset --write')} to write directly to the config`);
-        }
-        telemetry_1.default.record('command_used', {
-            name: 'generate_dataset',
-            numPrompts: testSuite.prompts.length,
-            numTestsExisting: (testSuite.tests || []).length,
-            numTestsGenerated: results.length,
-            duration: Math.round((Date.now() - startTime) / 1000),
-        });
-        await telemetry_1.default.send();
-    });
-    generateCommand
-        .command('redteam')
-        .description('Generate adversarial test cases')
-        .option('-c, --config [path]', 'Path to configuration file. Defaults to promptfooconfig.yaml')
-        .option('-o, --output [path]', 'Path to output file')
-        .option('-w, --write', 'Write results to promptfoo configuration file')
-        .option('--purpose <purpose>', 'Set the system purpose. If not set, the system purpose will be inferred from the config file')
-        .option('--provider <provider>', `Provider to use for generating adversarial tests. Defaults to: ${constants_1.REDTEAM_MODEL}`)
-        .option('--injectVar <varname>', 'Override the variable to inject user input into the prompt. If not set, the variable will default to {{query}}')
-        .option('--plugins <plugins>', (0, dedent_1.default) `Comma-separated list of plugins to use. Defaults to:
-        \n- ${Array.from(redteam_1.DEFAULT_PLUGINS).sort().join('\n- ')}\n\n
-    `, (val) => val.split(',').map((x) => x.trim()))
-        .option('--add-plugins <plugins>', (0, dedent_1.default) `Comma-separated list of plugins to run in addition to the default plugins:
-        \n- ${redteam_1.ADDITIONAL_PLUGINS.sort().join('\n- ')}\n\n
-      `, (val) => val.split(',').map((x) => x.trim()))
-        .option('--no-cache', 'Do not read or write results to disk cache', false)
-        .option('--env-file <path>', 'Path to .env file')
-        .action(async ({ addPlugins, cache, config, envFile, injectVar, output, plugins, provider, purpose, write, }) => {
-        (0, util_1.setupEnv)(envFile);
-        if (!cache) {
-            logger_1.default.info('Cache is disabled.');
-            (0, cache_1.disableCache)();
-        }
-        let testSuite;
-        const configPath = config || defaultConfigPath;
-        if (configPath) {
-            const resolved = await resolveConfigs({
-                config: [configPath],
-            }, defaultConfig);
-            testSuite = resolved.testSuite;
-        }
-        else {
-            throw new Error('Could not find config file. Please use `--config`');
-        }
-        const startTime = Date.now();
-        telemetry_1.default.record('command_used', {
-            name: 'generate redteam - started',
-            numPrompts: testSuite.prompts.length,
-            numTestsExisting: (testSuite.tests || []).length,
-        });
-        await telemetry_1.default.send();
-        const redteamTests = await (0, redteam_1.synthesizeFromTestSuite)(testSuite, {
-            purpose,
-            injectVar,
-            plugins: addPlugins && addPlugins.length > 0
-                ? Array.from(plugins || redteam_1.DEFAULT_PLUGINS).concat(addPlugins)
-                : plugins,
-            provider,
-        });
-        if (output) {
-            const existingYaml = js_yaml_1.default.load(fs_1.default.readFileSync(configPath, 'utf8'));
-            const updatedYaml = {
-                ...existingYaml,
-                tests: redteamTests,
-                metadata: {
-                    ...existingYaml.metadata,
-                    redteam: true,
-                },
-            };
-            fs_1.default.writeFileSync(output, js_yaml_1.default.dump(updatedYaml, { skipInvalid: true }));
-            (0, util_1.printBorder)();
-            logger_1.default.info(`Wrote ${redteamTests.length} new test cases to ${output}`);
-            (0, util_1.printBorder)();
-        }
-        else if (write && configPath) {
-            const existingConfig = js_yaml_1.default.load(fs_1.default.readFileSync(configPath, 'utf8'));
-            existingConfig.tests = [...(existingConfig.tests || []), ...redteamTests];
-            fs_1.default.writeFileSync(configPath, js_yaml_1.default.dump(existingConfig));
-            logger_1.default.info(`Wrote ${redteamTests.length} new test cases to ${configPath}`);
-        }
-        else {
-            logger_1.default.info(js_yaml_1.default.dump(redteamTests, { skipInvalid: true }));
-        }
-        telemetry_1.default.record('command_used', {
-            name: 'generate redteam',
-            numPrompts: testSuite.prompts.length,
-            numTestsExisting: (testSuite.tests || []).length,
-            numTestsGenerated: redteamTests.length,
-            duration: Math.round((Date.now() - startTime) / 1000),
-        });
-        await telemetry_1.default.send();
-    });
-    program
-        .command('eval')
-        .description('Evaluate prompts')
-        .option('-p, --prompts <paths...>', 'Paths to prompt files (.txt)')
-        .option('-r, --providers <name or path...>', 'One of: openai:chat, openai:completion, openai:<model name>, or path to custom API caller module')
-        .option('-c, --config <paths...>', 'Path to configuration file. Automatically loads promptfooconfig.js/json/yaml')
-        .option(
-    // TODO(ian): Remove `vars` for v1
-    '-v, --vars, -t, --tests <path>', 'Path to CSV with test cases', defaultConfig?.commandLineOptions?.vars)
-        .option('-a, --assertions <path>', 'Path to assertions file')
-        .option('--model-outputs <path>', 'Path to JSON containing list of LLM output strings')
-        .option('-t, --tests <path>', 'Path to CSV with test cases')
-        .option('-o, --output <paths...>', 'Path to output file (csv, txt, json, yaml, yml, html), default is no output file')
-        .option('-j, --max-concurrency <number>', 'Maximum number of concurrent API calls', defaultConfig.evaluateOptions?.maxConcurrency
-        ? String(defaultConfig.evaluateOptions.maxConcurrency)
-        : `${evaluator_1.DEFAULT_MAX_CONCURRENCY}`)
-        .option('--repeat <number>', 'Number of times to run each test', defaultConfig.evaluateOptions?.repeat ? String(defaultConfig.evaluateOptions.repeat) : '1')
-        .option('--delay <number>', 'Delay between each test (in milliseconds)', defaultConfig.evaluateOptions?.delay ? String(defaultConfig.evaluateOptions.delay) : '0')
-        .option('--table-cell-max-length <number>', 'Truncate console table cells to this length', '250')
-        .option('--suggest-prompts <number>', 'Generate N new prompts and append them to the prompt list')
-        .option('--prompt-prefix <path>', 'This prefix is prepended to every prompt', defaultConfig.defaultTest?.options?.prefix)
-        .option('--prompt-suffix <path>', 'This suffix is append to every prompt', defaultConfig.defaultTest?.options?.suffix)
-        .option('--no-write', 'Do not write results to promptfoo directory', defaultConfig?.commandLineOptions?.write)
-        .option('--no-cache', 'Do not read or write results to disk cache', 
-    // TODO(ian): Remove commandLineOptions.cache in v1
-    defaultConfig?.commandLineOptions?.cache ?? defaultConfig?.evaluateOptions?.cache)
-        .option('--no-progress-bar', 'Do not show progress bar')
-        .option('--table', 'Output table in CLI', defaultConfig?.commandLineOptions?.table ?? true)
-        .option('--no-table', 'Do not output table in CLI', defaultConfig?.commandLineOptions?.table)
-        .option('--share', 'Create a shareable URL', defaultConfig?.commandLineOptions?.share)
-        .option('--grader <provider>', 'Model that will grade outputs', defaultConfig?.commandLineOptions?.grader)
-        .option('--verbose', 'Show debug logs', defaultConfig?.commandLineOptions?.verbose)
-        .option('-w, --watch', 'Watch for changes in config and re-run')
-        .option('--env-file <path>', 'Path to .env file')
-        .option('--interactive-providers', 'Run providers interactively, one at a time', defaultConfig?.evaluateOptions?.interactiveProviders)
-        .option('-n, --filter-first-n <number>', 'Only run the first N tests')
-        .option('--filter-pattern <pattern>', 'Only run tests whose description matches the regular expression pattern')
-        .option('--filter-providers <providers>', 'Only run tests with these providers')
-        .option('--filter-failing <path>', 'Path to json output file')
-        .option('--var <key=value>', 'Set a variable in key=value format', (value, previous = {}) => {
-        const [key, val] = value.split('=');
-        if (!key || val === undefined) {
-            throw new Error('--var must be specified in key=value format.');
-        }
-        previous[key] = val;
-        return previous;
-    }, {})
-        .action(async (cmdObj) => {
-        (0, util_1.setupEnv)(cmdObj.envFile);
-        let config = undefined;
-        let testSuite = undefined;
-        let basePath = undefined;
-        const runEvaluation = async (initialization) => {
-            const startTime = Date.now();
-            telemetry_1.default.record('command_used', {
-                name: 'eval - started',
-                watch: Boolean(cmdObj.watch),
-            });
-            await telemetry_1.default.send();
-            // Misc settings
-            if (cmdObj.verbose) {
-                (0, logger_1.setLogLevel)('debug');
-            }
-            const iterations = parseInt(cmdObj.repeat || '', 10);
-            const repeat = !isNaN(iterations) && iterations > 0 ? iterations : 1;
-            if (!cmdObj.cache || repeat > 1) {
-                logger_1.default.info('Cache is disabled.');
-                (0, cache_1.disableCache)();
-            }
-            ({ config, testSuite, basePath } = await resolveConfigs(cmdObj, defaultConfig));
-            cliState_1.default.basePath = basePath;
-            let maxConcurrency = parseInt(cmdObj.maxConcurrency || '', 10);
-            const delay = parseInt(cmdObj.delay || '', 0);
-            if (delay > 0) {
-                maxConcurrency = 1;
-                logger_1.default.info(`Running at concurrency=1 because ${delay}ms delay was requested between API calls`);
-            }
-            testSuite.tests = await (0, filterTests_1.filterTests)(testSuite, {
-                firstN: cmdObj.filterFirstN,
-                pattern: cmdObj.filterPattern,
-                failing: cmdObj.filterFailing,
-            });
-            testSuite.providers = (0, filterProviders_1.filterProviders)(testSuite.providers, cmdObj.filterProviders);
-            const options = {
-                showProgressBar: (0, logger_1.getLogLevel)() === 'debug' ? false : cmdObj.progressBar,
-                maxConcurrency: !isNaN(maxConcurrency) && maxConcurrency > 0 ? maxConcurrency : undefined,
-                repeat,
-                delay: !isNaN(delay) && delay > 0 ? delay : undefined,
-                interactiveProviders: cmdObj.interactiveProviders,
-                ...evaluateOptions,
-            };
-            if (cmdObj.grader) {
-                testSuite.defaultTest = testSuite.defaultTest || {};
-                testSuite.defaultTest.options = testSuite.defaultTest.options || {};
-                testSuite.defaultTest.options.provider = await (0, providers_1.loadApiProvider)(cmdObj.grader);
-            }
-            if (cmdObj.var) {
-                testSuite.defaultTest = testSuite.defaultTest || {};
-                testSuite.defaultTest.vars = { ...testSuite.defaultTest.vars, ...cmdObj.var };
-            }
-            if (cmdObj.generateSuggestions) {
-                options.generateSuggestions = true;
-            }
-            const summary = await (0, evaluator_1.evaluate)(testSuite, {
-                ...options,
-                eventSource: 'cli',
-            });
-            const shareableUrl = cmdObj.share && config.sharing ? await (0, share_1.createShareableUrl)(summary, config) : null;
-            if (cmdObj.table && (0, logger_1.getLogLevel)() !== 'debug') {
-                // Output CLI table
-                const table = (0, table_1.generateTable)(summary, parseInt(cmdObj.tableCellMaxLength || '', 10));
-                logger_1.default.info('\n' + table.toString());
-                if (summary.table.body.length > 25) {
-                    const rowsLeft = summary.table.body.length - 25;
-                    logger_1.default.info(`... ${rowsLeft} more row${rowsLeft === 1 ? '' : 's'} not shown ...\n`);
-                }
-            }
-            else if (summary.stats.failures !== 0) {
-                logger_1.default.debug(`At least one evaluation failure occurred. This might be caused by the underlying call to the provider, or a test failure. Context: \n${JSON.stringify(summary.results)}`);
-            }
-            await (0, util_1.migrateResultsFromFileSystemToDatabase)();
-            let evalId = null;
-            if (cmdObj.write) {
-                evalId = await (0, util_1.writeResultsToDatabase)(summary, config);
-            }
-            const { outputPath } = config;
-            if (outputPath) {
-                // Write output to file
-                if (typeof outputPath === 'string') {
-                    await (0, util_1.writeOutput)(outputPath, evalId, summary, config, shareableUrl);
-                }
-                else if (Array.isArray(outputPath)) {
-                    await (0, util_1.writeMultipleOutputs)(outputPath, evalId, summary, config, shareableUrl);
-                }
-                logger_1.default.info(chalk_1.default.yellow(`Writing output to ${outputPath}`));
-            }
-            telemetry_1.default.maybeShowNotice();
-            (0, util_1.printBorder)();
-            if (!cmdObj.write) {
-                logger_1.default.info(`${chalk_1.default.green('✔')} Evaluation complete`);
-            }
-            else {
-                if (shareableUrl) {
-                    logger_1.default.info(`${chalk_1.default.green('✔')} Evaluation complete: ${shareableUrl}`);
-                }
-                else {
-                    logger_1.default.info(`${chalk_1.default.green('✔')} Evaluation complete.\n`);
-                    logger_1.default.info(`» Run ${chalk_1.default.greenBright.bold('promptfoo view')} to use the local web viewer`);
-                    logger_1.default.info(`» Run ${chalk_1.default.greenBright.bold('promptfoo share')} to create a shareable URL`);
-                    logger_1.default.info(`» This project needs your feedback. What's one thing we can improve? ${chalk_1.default.greenBright.bold('https://forms.gle/YFLgTe1dKJKNSCsU7')}`);
-                }
-            }
-            (0, util_1.printBorder)();
-            logger_1.default.info(chalk_1.default.green.bold(`Successes: ${summary.stats.successes}`));
-            logger_1.default.info(chalk_1.default.red.bold(`Failures: ${summary.stats.failures}`));
-            logger_1.default.info(`Token usage: Total ${summary.stats.tokenUsage.total}, Prompt ${summary.stats.tokenUsage.prompt}, Completion ${summary.stats.tokenUsage.completion}, Cached ${summary.stats.tokenUsage.cached}`);
-            telemetry_1.default.record('command_used', {
-                name: 'eval',
-                watch: Boolean(cmdObj.watch),
-                duration: Math.round((Date.now() - startTime) / 1000),
-            });
-            await telemetry_1.default.send();
-            if (cmdObj.watch) {
-                if (initialization) {
-                    const configPaths = (cmdObj.config || [defaultConfigPath]).filter(Boolean);
-                    if (!configPaths.length) {
-                        logger_1.default.error('Could not locate config file(s) to watch');
-                        process.exit(1);
-                    }
-                    const basePath = path_1.default.dirname(configPaths[0]);
-                    const promptPaths = Array.isArray(config.prompts)
-                        ? config.prompts
-                            .map((p) => {
-                            if (typeof p === 'string' && p.startsWith('file://')) {
-                                return path_1.default.resolve(basePath, p.slice('file://'.length));
-                            }
-                            else if (typeof p === 'object' && p.id && p.id.startsWith('file://')) {
-                                return path_1.default.resolve(basePath, p.id.slice('file://'.length));
-                            }
-                            return null;
-                        })
-                            .filter(Boolean)
-                        : [];
-                    const providerPaths = Array.isArray(config.providers)
-                        ? config.providers
-                            .map((p) => typeof p === 'string' && p.startsWith('file://')
-                            ? path_1.default.resolve(basePath, p.slice('file://'.length))
-                            : null)
-                            .filter(Boolean)
-                        : [];
-                    const varPaths = Array.isArray(config.tests)
-                        ? config.tests
-                            .flatMap((t) => {
-                            if (typeof t === 'string' && t.startsWith('file://')) {
-                                return path_1.default.resolve(basePath, t.slice('file://'.length));
-                            }
-                            else if (typeof t !== 'string' && t.vars) {
-                                return Object.values(t.vars).flatMap((v) => {
-                                    if (typeof v === 'string' && v.startsWith('file://')) {
-                                        return path_1.default.resolve(basePath, v.slice('file://'.length));
-                                    }
-                                    return [];
-                                });
-                            }
-                            return [];
-                        })
-                            .filter(Boolean)
-                        : [];
-                    const watchPaths = Array.from(new Set([...configPaths, ...promptPaths, ...providerPaths, ...varPaths]));
-                    const watcher = chokidar_1.default.watch(watchPaths, { ignored: /^\./, persistent: true });
-                    watcher
-                        .on('change', async (path) => {
-                        (0, util_1.printBorder)();
-                        logger_1.default.info(`File change detected: ${path}`);
-                        (0, util_1.printBorder)();
-                        await runEvaluation();
-                    })
-                        .on('error', (error) => logger_1.default.error(`Watcher error: ${error}`))
-                        .on('ready', () => watchPaths.forEach((watchPath) => logger_1.default.info(`Watching for file changes on ${watchPath} ...`)));
-                }
-            }
-            else {
-                logger_1.default.info('Done.');
-                if (summary.stats.failures > 0) {
-                    const exitCode = Number(process.env.PROMPTFOO_FAILED_TEST_EXIT_CODE);
-                    process.exit(isNaN(exitCode) ? 100 : exitCode);
-                }
-            }
-        };
-        await runEvaluation(true /* initialization */);
-    });
-    (0, list_1.listCommand)(program);
-    (0, show_1.showCommand)(program);
+    (0, config_1.configCommand)(program);
     (0, delete_1.deleteCommand)(program);
-    (0, import_1.importCommand)(program);
+    (0, eval_1.evalCommand)(program, defaultConfig, defaultConfigPath, evaluateOptions);
     (0, export_1.exportCommand)(program);
-    (0, config_1.configCommand)(program);
+    (0, generate_1.generateCommand)(program, defaultConfig, defaultConfigPath);
+    (0, import_1.importCommand)(program);
+    (0, list_1.listCommand)(program);
+    (0, redteam_1.redteamCommand)(program);
+    (0, show_1.showCommand)(program);
     program.parse(process.argv);
     if (!process.argv.slice(2).length) {
         program.outputHelp();