promptfoo 0.69.2 → 0.70.1

package/dist/src/commands/generate.d.ts

@@ -0,0 +1,20 @@
+import { Command } from 'commander';
+import { UnifiedConfig } from '../types';
+interface RedteamGenerateOptions {
+    addPlugins?: string[];
+    cache: boolean;
+    config?: string;
+    envFile?: string;
+    injectVar?: string;
+    output?: string;
+    plugins?: string[];
+    provider?: string;
+    purpose?: string;
+    write: boolean;
+    defaultConfig: Partial<UnifiedConfig>;
+    defaultConfigPath: string | undefined;
+}
+export declare function doGenerateRedteam({ addPlugins, cache, config, envFile, injectVar, output, plugins, provider, purpose, write, defaultConfig, defaultConfigPath, }: RedteamGenerateOptions): Promise<void>;
+export declare function generateCommand(program: Command, defaultConfig: Partial<UnifiedConfig>, defaultConfigPath: string | undefined): void;
+export {};
+//# sourceMappingURL=generate.d.ts.map

package/dist/src/commands/generate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../src/commands/generate.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAepC,OAAO,EAAa,aAAa,EAAE,MAAM,UAAU,CAAC;AAGpD,UAAU,sBAAsB;IAE9B,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;IAGf,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACtC,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;CACvC;AAED,wBAAsB,iBAAiB,CAAC,EACtC,UAAU,EACV,KAAK,EACL,MAAM,EACN,OAAO,EACP,SAAS,EACT,MAAM,EACN,OAAO,EACP,QAAQ,EACR,OAAO,EACP,KAAK,EACL,aAAa,EACb,iBAAiB,GAClB,EAAE,sBAAsB,iBAuFxB;AAED,wBAAgB,eAAe,CAC7B,OAAO,EAAE,OAAO,EAChB,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,EACrC,iBAAiB,EAAE,MAAM,GAAG,SAAS,QA6ItC"}

package/dist/src/commands/generate.js

@@ -0,0 +1,215 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.doGenerateRedteam = doGenerateRedteam;
+exports.generateCommand = generateCommand;
+const chalk_1 = __importDefault(require("chalk"));
+const dedent_1 = __importDefault(require("dedent"));
+const fs = __importStar(require("fs"));
+const js_yaml_1 = __importDefault(require("js-yaml"));
+const cache_1 = require("../cache");
+const config_1 = require("../config");
+const logger_1 = __importDefault(require("../logger"));
+const redteam_1 = require("../redteam");
+const constants_1 = require("../redteam/constants");
+const telemetry_1 = __importDefault(require("../telemetry"));
+const testCases_1 = require("../testCases");
+const util_1 = require("../util");
+async function doGenerateRedteam({ addPlugins, cache, config, envFile, injectVar, output, plugins, provider, purpose, write, defaultConfig, defaultConfigPath, }) {
+    (0, util_1.setupEnv)(envFile);
+    if (!cache) {
+        logger_1.default.info('Cache is disabled.');
+        (0, cache_1.disableCache)();
+    }
+    let testSuite;
+    const configPath = config || defaultConfigPath;
+    if (configPath) {
+        const resolved = await (0, config_1.resolveConfigs)({
+            config: [configPath],
+        }, defaultConfig);
+        testSuite = resolved.testSuite;
+    }
+    else if (purpose) {
+        // There is a purpose, so we can just have a dummy testsuite for standalone invocation
+        testSuite = {
+            prompts: [],
+            providers: [],
+            tests: [],
+        };
+    }
+    else {
+        logger_1.default.info(chalk_1.default.red(`\nCan't generate without configuration - run ${chalk_1.default.yellow.bold('promptfoo redteam init')} first`));
+        return;
+    }
+    const startTime = Date.now();
+    telemetry_1.default.record('command_used', {
+        name: 'generate redteam - started',
+        numPrompts: testSuite.prompts.length,
+        numTestsExisting: (testSuite.tests || []).length,
+    });
+    await telemetry_1.default.send();
+    const redteamTests = await (0, redteam_1.synthesizeFromTestSuite)(testSuite, {
+        purpose,
+        injectVar,
+        plugins: addPlugins && addPlugins.length > 0
+            ? Array.from(plugins || constants_1.DEFAULT_PLUGINS).concat(addPlugins)
+            : plugins,
+        provider,
+    });
+    if (output) {
+        const existingYaml = configPath
+            ? js_yaml_1.default.load(fs.readFileSync(configPath, 'utf8'))
+            : {};
+        const updatedYaml = {
+            ...existingYaml,
+            tests: redteamTests,
+            metadata: {
+                ...existingYaml.metadata,
+                redteam: true,
+            },
+        };
+        fs.writeFileSync(output, js_yaml_1.default.dump(updatedYaml, { skipInvalid: true }));
+        (0, util_1.printBorder)();
+        logger_1.default.info(`Wrote ${redteamTests.length} new test cases to ${output}`);
+        (0, util_1.printBorder)();
+    }
+    else if (write && configPath) {
+        const existingConfig = js_yaml_1.default.load(fs.readFileSync(configPath, 'utf8'));
+        existingConfig.tests = [...(existingConfig.tests || []), ...redteamTests];
+        fs.writeFileSync(configPath, js_yaml_1.default.dump(existingConfig));
+        logger_1.default.info(`\nWrote ${redteamTests.length} new test cases to ${configPath}`);
+        logger_1.default.info('\n' + chalk_1.default.green(`Run ${chalk_1.default.bold('promptfoo eval')} to run the generated tests`));
+    }
+    else {
+        logger_1.default.info(js_yaml_1.default.dump(redteamTests, { skipInvalid: true }));
+    }
+    telemetry_1.default.record('command_used', {
+        name: 'generate redteam',
+        numPrompts: testSuite.prompts.length,
+        numTestsExisting: (testSuite.tests || []).length,
+        numTestsGenerated: redteamTests.length,
+        duration: Math.round((Date.now() - startTime) / 1000),
+    });
+    await telemetry_1.default.send();
+}
+function generateCommand(program, defaultConfig, defaultConfigPath) {
+    const generateCommand = program.command('generate').description('Generate synthetic data');
+    generateCommand
+        .command('dataset')
+        .description('Generate test cases')
+        .option('-i, --instructions [instructions]', 'Additional instructions to follow while generating test cases')
+        .option('-c, --config [path]', 'Path to configuration file. Defaults to promptfooconfig.yaml')
+        .option('-o, --output [path]', 'Path to output file')
+        .option('-w, --write', 'Write results to promptfoo configuration file')
+        .option('--numPersonas <number>', 'Number of personas to generate', '5')
+        .option('--numTestCasesPerPersona <number>', 'Number of test cases per persona', '3')
+        .option('--no-cache', 'Do not read or write results to disk cache', false)
+        .option('--env-file <path>', 'Path to .env file')
+        .action(async (options) => {
+        (0, util_1.setupEnv)(options.envFile);
+        if (!options.cache) {
+            logger_1.default.info('Cache is disabled.');
+            (0, cache_1.disableCache)();
+        }
+        let testSuite;
+        const configPath = options.config || defaultConfigPath;
+        if (configPath) {
+            const resolved = await (0, config_1.resolveConfigs)({
+                config: [configPath],
+            }, defaultConfig);
+            testSuite = resolved.testSuite;
+        }
+        else {
+            throw new Error('Could not find config file. Please use `--config`');
+        }
+        const startTime = Date.now();
+        telemetry_1.default.record('command_used', {
+            name: 'generate_dataset - started',
+            numPrompts: testSuite.prompts.length,
+            numTestsExisting: (testSuite.tests || []).length,
+        });
+        await telemetry_1.default.send();
+        const results = await (0, testCases_1.synthesizeFromTestSuite)(testSuite, {
+            instructions: options.instructions,
+            numPersonas: parseInt(options.numPersonas, 10),
+            numTestCasesPerPersona: parseInt(options.numTestCasesPerPersona, 10),
+        });
+        const configAddition = { tests: results.map((result) => ({ vars: result })) };
+        const yamlString = js_yaml_1.default.dump(configAddition);
+        if (options.output) {
+            fs.writeFileSync(options.output, yamlString);
+            (0, util_1.printBorder)();
+            logger_1.default.info(`Wrote ${results.length} new test cases to ${options.output}`);
+            (0, util_1.printBorder)();
+        }
+        else {
+            (0, util_1.printBorder)();
+            logger_1.default.info('New test Cases');
+            (0, util_1.printBorder)();
+            logger_1.default.info(yamlString);
+        }
+        (0, util_1.printBorder)();
+        if (options.write && configPath) {
+            const existingConfig = js_yaml_1.default.load(fs.readFileSync(configPath, 'utf8'));
+            existingConfig.tests = [...(existingConfig.tests || []), ...configAddition.tests];
+            fs.writeFileSync(configPath, js_yaml_1.default.dump(existingConfig));
+            logger_1.default.info(`Wrote ${results.length} new test cases to ${configPath}`);
+            logger_1.default.info(chalk_1.default.green(`Run ${chalk_1.default.bold('promptfoo eval')} to run the generated tests`));
+        }
+        else {
+            logger_1.default.info(`Copy the above test cases or run ${chalk_1.default.greenBright('promptfoo generate dataset --write')} to write directly to the config`);
+        }
+        telemetry_1.default.record('command_used', {
+            name: 'generate_dataset',
+            numPrompts: testSuite.prompts.length,
+            numTestsExisting: (testSuite.tests || []).length,
+            numTestsGenerated: results.length,
+            duration: Math.round((Date.now() - startTime) / 1000),
+        });
+        await telemetry_1.default.send();
+    });
+    generateCommand
+        .command('redteam')
+        .description('Generate adversarial test cases')
+        .option('-c, --config [path]', 'Path to configuration file. Defaults to promptfooconfig.yaml')
+        .option('-o, --output [path]', 'Path to output file')
+        .option('-w, --write', 'Write results to promptfoo configuration file')
+        .option('--purpose <purpose>', 'Set the system purpose. If not set, the system purpose will be inferred from the config file')
+        .option('--provider <provider>', `Provider to use for generating adversarial tests. Defaults to: ${constants_1.REDTEAM_MODEL}`)
+        .option('--injectVar <varname>', 'Override the variable to inject user input into the prompt. If not set, the variable will default to {{query}}')
+        .option('--plugins <plugins>', (0, dedent_1.default) `Comma-separated list of plugins to use. Defaults to:
+        \n- ${Array.from(constants_1.DEFAULT_PLUGINS).sort().join('\n- ')}\n\n
+    `, (val) => val.split(',').map((x) => x.trim()))
+        .option('--add-plugins <plugins>', (0, dedent_1.default) `Comma-separated list of plugins to run in addition to the default plugins:
+        \n- ${constants_1.ADDITIONAL_PLUGINS.sort().join('\n- ')}\n\n
+      `, (val) => val.split(',').map((x) => x.trim()))
+        .option('--no-cache', 'Do not read or write results to disk cache', false)
+        .option('--env-file <path>', 'Path to .env file')
+        .action((opts) => doGenerateRedteam({ ...opts, defaultConfig, defaultConfigPath }));
+}
+//# sourceMappingURL=generate.js.map

package/dist/src/commands/generate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../../src/commands/generate.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqCA,8CAoGC;AAED,0CAgJC;AA3RD,kDAA0B;AAE1B,oDAA4B;AAC5B,uCAAyB;AACzB,sDAA2B;AAC3B,oCAAwC;AACxC,sCAA2C;AAC3C,uDAA+B;AAC/B,wCAAuF;AACvF,oDAI8B;AAC9B,6DAAqC;AACrC,4CAAuD;AAEvD,kCAAgD;AAoBzC,KAAK,UAAU,iBAAiB,CAAC,EACtC,UAAU,EACV,KAAK,EACL,MAAM,EACN,OAAO,EACP,SAAS,EACT,MAAM,EACN,OAAO,EACP,QAAQ,EACR,OAAO,EACP,KAAK,EACL,aAAa,EACb,iBAAiB,GACM;IACvB,IAAA,eAAQ,EAAC,OAAO,CAAC,CAAC;IAClB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,gBAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAClC,IAAA,oBAAY,GAAE,CAAC;IACjB,CAAC;IAED,IAAI,SAAoB,CAAC;IACzB,MAAM,UAAU,GAAG,MAAM,IAAI,iBAAiB,CAAC;IAC/C,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,QAAQ,GAAG,MAAM,IAAA,uBAAc,EACnC;YACE,MAAM,EAAE,CAAC,UAAU,CAAC;SACrB,EACD,aAAa,CACd,CAAC;QACF,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC;IACjC,CAAC;SAAM,IAAI,OAAO,EAAE,CAAC;QACnB,sFAAsF;QACtF,SAAS,GAAG;YACV,OAAO,EAAE,EAAE;YACX,SAAS,EAAE,EAAE;YACb,KAAK,EAAE,EAAE;SACV,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,GAAG,CACP,gDAAgD,eAAK,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CACpG,CACF,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QAC/B,IAAI,EAAE,4BAA4B;QAClC,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM;QACpC,gBAAgB,EAAE,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM;KACjD,CAAC,CAAC;IACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;IAEvB,MAAM,YAAY,GAAG,MAAM,IAAA,iCAA8B,EAAC,SAAS,EAAE;QACnE,OAAO;QACP,SAAS;QACT,OAAO,EACL,UAAU,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;YACjC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,2BAAuB,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC;YACnE,CAAC,CAAC,OAAO;QACb,QAAQ;KACT,CAAC,CAAC;IAEH,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,YAAY,GAAG,UAAU;YAC7B,CAAC,CAAE,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAA4B;YAC5E,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,WAAW,GAAG;YAClB,GAAG,YAAY;YACf,KAAK,EAAE,YAAY;YACnB,QAAQ,EAAE;gBACR,GAAG,YAAY,CAAC,QAAQ;gBACxB,OAAO,EAAE,IAAI;aACd;SACF,CAAC;QACF,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE,iBAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACxE,IAAA,kBAAW,GAAE,CAAC;QACd,gBAAM,CAAC,IAAI,CAAC,SAAS,YAAY,CAAC,MAAM,sBAAsB,MAAM,EAAE,CAAC,CAAC;QACxE,IAAA,kBAAW,GAAE,CAAC;IAChB,CAAC;SAAM,IAAI,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,cAAc,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAA2B,CAAC;QAChG,cAAc,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,GAAG,YAAY,CAAC,CAAC;QAC1E,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,iBAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;QACxD,gBAAM,CAAC,IAAI,CAAC,WAAW,YAAY,CAAC,MAAM,sBAAsB,UAAU,EAAE,CAAC,CAAC;QAC9E,gBAAM,CAAC,IAAI,CACT,IAAI,GAAG,eAAK,CAAC,KAAK,CAAC,OAAO,eAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,6BAA6B,CAAC,CACrF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,gBAAM,CAAC,IAAI,CAAC,iBAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QAC/B,IAAI,EAAE,kBAAkB;QACxB,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM;QACpC,gBAAgB,EAAE,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM;QAChD,iBAAiB,EAAE,YAAY,CAAC,MAAM;QACtC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;KACtD,CAAC,CAAC;IACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAED,SAAgB,eAAe,CAC7B,OAAgB,EAChB,aAAqC,EACrC,iBAAqC;IAErC,MAAM,eAAe,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,WAAW,CAAC,yBAAyB,CAAC,CAAC;IAE3F,eAAe;SACZ,OAAO,CAAC,SAAS,CAAC;SAClB,WAAW,CAAC,qBAAqB,CAAC;SAClC,MAAM,CACL,mCAAmC,EACnC,+DAA+D,CAChE;SACA,MAAM,CAAC,qBAAqB,EAAE,8DAA8D,CAAC;SAC7F,MAAM,CAAC,qBAAqB,EAAE,qBAAqB,CAAC;SACpD,MAAM,CAAC,aAAa,EAAE,+CAA+C,CAAC;SACtE,MAAM,CAAC,wBAAwB,EAAE,gCAAgC,EAAE,GAAG,CAAC;SACvE,MAAM,CAAC,mCAAmC,EAAE,kCAAkC,EAAE,GAAG,CAAC;SACpF,MAAM,CAAC,YAAY,EAAE,4CAA4C,EAAE,KAAK,CAAC;SACzE,MAAM,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;SAChD,MAAM,CACL,KAAK,EAAE,OASN,EAAE,EAAE;QACH,IAAA,eAAQ,EAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC1B,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,gBAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAClC,IAAA,oBAAY,GAAE,CAAC;QACjB,CAAC;QAED,IAAI,SAAoB,CAAC;QACzB,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,IAAI,iBAAiB,CAAC;QACvD,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,MAAM,IAAA,uBAAc,EACnC;gBACE,MAAM,EAAE,CAAC,UAAU,CAAC;aACrB,EACD,aAAa,CACd,CAAC;YACF,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QACvE,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,4BAA4B;YAClC,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM;YACpC,gBAAgB,EAAE,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM;SACjD,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,MAAM,OAAO,GAAG,MAAM,IAAA,mCAAuB,EAAC,SAAS,EAAE;YACvD,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;YAC9C,sBAAsB,EAAE,QAAQ,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC;SACrE,CAAC,CAAC;QACH,MAAM,cAAc,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC;QAC9E,MAAM,UAAU,GAAG,iBAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC7C,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;YAC7C,IAAA,kBAAW,GAAE,CAAC;YACd,gBAAM,CAAC,IAAI,CAAC,SAAS,OAAO,CAAC,MAAM,sBAAsB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAC3E,IAAA,kBAAW,GAAE,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,IAAA,kBAAW,GAAE,CAAC;YACd,gBAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAC9B,IAAA,kBAAW,GAAE,CAAC;YACd,gBAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC1B,CAAC;QAED,IAAA,kBAAW,GAAE,CAAC;QACd,IAAI,OAAO,CAAC,KAAK,IAAI,UAAU,EAAE,CAAC;YAChC,MAAM,cAAc,GAAG,iBAAI,CAAC,IAAI,CAC9B,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CACV,CAAC;YAC5B,cAAc,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YAClF,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,iBAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;YACxD,gBAAM,CAAC,IAAI,CAAC,SAAS,OAAO,CAAC,MAAM,sBAAsB,UAAU,EAAE,CAAC,CAAC;YACvE,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,KAAK,CAAC,OAAO,eAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,6BAA6B,CAAC,CAC9E,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,gBAAM,CAAC,IAAI,CACT,oCAAoC,eAAK,CAAC,WAAW,CACnD,oCAAoC,CACrC,kCAAkC,CACpC,CAAC;QACJ,CAAC;QAED,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,kBAAkB;YACxB,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM;YACpC,gBAAgB,EAAE,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM;YAChD,iBAAiB,EAAE,OAAO,CAAC,MAAM;YACjC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;SACtD,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC,CACF,CAAC;IAEJ,eAAe;SACZ,OAAO,CAAC,SAAS,CAAC;SAClB,WAAW,CAAC,iCAAiC,CAAC;SAC9C,MAAM,CAAC,qBAAqB,EAAE,8DAA8D,CAAC;SAC7F,MAAM,CAAC,qBAAqB,EAAE,qBAAqB,CAAC;SACpD,MAAM,CAAC,aAAa,EAAE,+CAA+C,CAAC;SACtE,MAAM,CACL,qBAAqB,EACrB,8FAA8F,CAC/F;SACA,MAAM,CACL,uBAAuB,EACvB,kEAAkE,yBAAa,EAAE,CAClF;SACA,MAAM,CACL,uBAAuB,EACvB,gHAAgH,CACjH;SACA,MAAM,CACL,qBAAqB,EACrB,IAAA,gBAAM,EAAA;cACE,KAAK,CAAC,IAAI,CAAC,2BAAuB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;KAChE,EACC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAC7C;SACA,MAAM,CACL,yBAAyB,EACzB,IAAA,gBAAM,EAAA;cACE,8BAA0B,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;OACrD,EACD,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAC7C;SACA,MAAM,CAAC,YAAY,EAAE,4CAA4C,EAAE,KAAK,CAAC;SACzE,MAAM,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;SAChD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,GAAG,IAAI,EAAE,aAAa,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC;AACxF,CAAC"}

package/dist/src/commands/redteam.d.ts

@@ -0,0 +1,10 @@
+import { Command } from 'commander';
+interface RunRedteamOptions {
+    config: string;
+    cache: boolean;
+    envFile: string;
+}
+export declare function doRunRedteam(cmdObj: RunRedteamOptions): Promise<void>;
+export declare function redteamCommand(program: Command): void;
+export {};
+//# sourceMappingURL=redteam.d.ts.map

package/dist/src/commands/redteam.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/commands/redteam.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAUpC,UAAU,iBAAiB;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,wBAAsB,YAAY,CAAC,MAAM,EAAE,iBAAiB,iBAAI;AAEhE,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,QA0K9C"}

package/dist/src/commands/redteam.js

@@ -0,0 +1,191 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.doRunRedteam = doRunRedteam;
+exports.redteamCommand = redteamCommand;
+const chalk_1 = __importDefault(require("chalk"));
+const fs = __importStar(require("fs"));
+const inquirer_1 = __importDefault(require("inquirer"));
+const js_yaml_1 = __importDefault(require("js-yaml"));
+const path = __importStar(require("path"));
+const logger_1 = __importDefault(require("../logger"));
+const constants_1 = require("../redteam/constants");
+const telemetry_1 = __importDefault(require("../telemetry"));
+const generate_1 = require("./generate");
+async function doRunRedteam(cmdObj) { }
+function redteamCommand(program) {
+    const redteamCommand = program.command('redteam').description('Red team LLM applications');
+    redteamCommand
+        .command('init [directory]')
+        .description('Initialize red teaming project')
+        .action(async (directory) => {
+        telemetry_1.default.maybeShowNotice();
+        telemetry_1.default.record('command_used', {
+            name: 'redteam init - started',
+        });
+        await telemetry_1.default.send();
+        let projectDir = directory;
+        if (!projectDir) {
+            const { chosenDir } = await inquirer_1.default.prompt([
+                {
+                    type: 'input',
+                    name: 'chosenDir',
+                    message: 'Where do you want to create the project?',
+                    default: '.',
+                },
+            ]);
+            projectDir = chosenDir || '.';
+        }
+        if (projectDir !== '.' && !fs.existsSync(projectDir)) {
+            fs.mkdirSync(projectDir, { recursive: true });
+        }
+        // Question 2: Prompt
+        const { promptChoice } = await inquirer_1.default.prompt([
+            {
+                type: 'list',
+                name: 'promptChoice',
+                message: 'How would you like to specify the prompt?',
+                choices: [
+                    { name: 'Enter a prompt', value: 'enter' },
+                    { name: 'Reference a prompt file', value: 'file' },
+                ],
+            },
+        ]);
+        let prompt;
+        if (promptChoice === 'enter') {
+            const { enteredPrompt } = await inquirer_1.default.prompt([
+                {
+                    type: 'editor',
+                    name: 'enteredPrompt',
+                    message: 'Enter your prompt:',
+                    default: 'You are a helpful concise assistant.\n\nUser query: {{query}}\n\n(NOTE: your prompt must include "{{query}}" as a placeholder for user input)',
+                },
+            ]);
+            prompt = enteredPrompt;
+        }
+        else {
+            const { promptFile } = await inquirer_1.default.prompt([
+                {
+                    type: 'input',
+                    name: 'promptFile',
+                    message: 'Enter the path to your prompt file (text or JSON):',
+                },
+            ]);
+            prompt = `file://${promptFile}`;
+        }
+        // Question 3: Provider
+        let provider;
+        const { providerChoice } = await inquirer_1.default.prompt([
+            {
+                type: 'list',
+                name: 'providerChoice',
+                message: 'Choose a provider:',
+                choices: [
+                    'openai:gpt-3.5-turbo',
+                    'openai:gpt-4',
+                    'anthropic:messages:claude-3-5-sonnet-20240620',
+                    'anthropic:messages:claude-3-opus-20240307',
+                    'vertex:gemini-pro',
+                    'Other',
+                ],
+            },
+        ]);
+        if (providerChoice === 'Other') {
+            const { customProvider } = await inquirer_1.default.prompt([
+                {
+                    type: 'input',
+                    name: 'customProvider',
+                    message: 'Enter the provider ID (see https://www.promptfoo.dev/docs/providers/ for options):',
+                },
+            ]);
+            provider = customProvider;
+        }
+        else {
+            provider = providerChoice;
+        }
+        // Question 4: Plugins
+        const pluginChoices = Array.from(constants_1.ALL_PLUGINS).map((plugin) => ({
+            name: `${plugin} - ${constants_1.subCategoryDescriptions[plugin] || 'No description available'}`,
+            value: plugin,
+            checked: constants_1.DEFAULT_PLUGINS.has(plugin),
+        }));
+        const { selectedPlugins } = await inquirer_1.default.prompt([
+            {
+                type: 'checkbox',
+                name: 'selectedPlugins',
+                message: 'Select the plugins you want to enable:',
+                choices: pluginChoices,
+                pageSize: 20,
+            },
+        ]);
+        const plugins = selectedPlugins;
+        // Create config file
+        const config = {
+            prompts: [prompt],
+            providers: [provider],
+            tests: [],
+        };
+        const configPath = path.join(projectDir, 'promptfooconfig.yaml');
+        fs.writeFileSync(configPath, js_yaml_1.default.dump(config), 'utf8');
+        logger_1.default.info('\n' + chalk_1.default.green(`Created red teaming configuration file at ${configPath}`) + '\n');
+        const { readyToGenerate } = await inquirer_1.default.prompt([
+            {
+                type: 'confirm',
+                name: 'readyToGenerate',
+                message: 'Are you ready to generate adversarial test cases?',
+                default: true,
+            },
+        ]);
+        telemetry_1.default.record('command_used', {
+            name: 'redteam init',
+        });
+        await telemetry_1.default.send();
+        if (readyToGenerate) {
+            await (0, generate_1.doGenerateRedteam)({
+                plugins,
+                cache: false,
+                write: true,
+                defaultConfig: config,
+                defaultConfigPath: configPath,
+            });
+        }
+        else {
+            logger_1.default.info('\n' +
+                chalk_1.default.blue('To generate test cases later, use the command: ' +
+                    chalk_1.default.bold('promptfoo generate redteam')));
+        }
+    });
+    redteamCommand
+        .command('run')
+        .description('Run red teaming evaluation')
+        .option('-c, --config <path>', 'Path to configuration file')
+        .option('--no-cache', 'Disable cache', false)
+        .option('--env-file <path>', 'Path to .env file')
+        .action(doRunRedteam);
+}
+//# sourceMappingURL=redteam.js.map

package/dist/src/commands/redteam.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"redteam.js","sourceRoot":"","sources":["../../../src/commands/redteam.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiBA,oCAAgE;AAEhE,wCA0KC;AA7LD,kDAA0B;AAE1B,uCAAyB;AACzB,wDAAgC;AAChC,sDAA2B;AAC3B,2CAA6B;AAC7B,uDAA+B;AAC/B,oDAA6F;AAC7F,6DAAqC;AACrC,yCAA+C;AAQxC,KAAK,UAAU,YAAY,CAAC,MAAyB,IAAG,CAAC;AAEhE,SAAgB,cAAc,CAAC,OAAgB;IAC7C,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,WAAW,CAAC,2BAA2B,CAAC,CAAC;IAE3F,cAAc;SACX,OAAO,CAAC,kBAAkB,CAAC;SAC3B,WAAW,CAAC,gCAAgC,CAAC;SAC7C,MAAM,CAAC,KAAK,EAAE,SAA6B,EAAE,EAAE;QAC9C,mBAAS,CAAC,eAAe,EAAE,CAAC;QAC5B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,wBAAwB;SAC/B,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,IAAI,UAAU,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,kBAAQ,CAAC,MAAM,CAAC;gBAC1C;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,0CAA0C;oBACnD,OAAO,EAAE,GAAG;iBACb;aACF,CAAC,CAAC;YACH,UAAU,GAAI,SAAgC,IAAI,GAAG,CAAC;QACxD,CAAC;QACD,IAAI,UAAU,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YACrD,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,CAAC;QAED,qBAAqB;QACrB,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,kBAAQ,CAAC,MAAM,CAAC;YAC7C;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,cAAc;gBACpB,OAAO,EAAE,2CAA2C;gBACpD,OAAO,EAAE;oBACP,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,OAAO,EAAE;oBAC1C,EAAE,IAAI,EAAE,yBAAyB,EAAE,KAAK,EAAE,MAAM,EAAE;iBACnD;aACF;SACF,CAAC,CAAC;QAEH,IAAI,MAAc,CAAC;QACnB,IAAI,YAAY,KAAK,OAAO,EAAE,CAAC;YAC7B,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,kBAAQ,CAAC,MAAM,CAAC;gBAC9C;oBACE,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,eAAe;oBACrB,OAAO,EAAE,oBAAoB;oBAC7B,OAAO,EACL,+IAA+I;iBAClJ;aACF,CAAC,CAAC;YACH,MAAM,GAAG,aAAa,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,kBAAQ,CAAC,MAAM,CAAC;gBAC3C;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,YAAY;oBAClB,OAAO,EAAE,oDAAoD;iBAC9D;aACF,CAAC,CAAC;YACH,MAAM,GAAG,UAAU,UAAU,EAAE,CAAC;QAClC,CAAC;QAED,uBAAuB;QACvB,IAAI,QAAgB,CAAC;QACrB,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,kBAAQ,CAAC,MAAM,CAAC;YAC/C;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,gBAAgB;gBACtB,OAAO,EAAE,oBAAoB;gBAC7B,OAAO,EAAE;oBACP,sBAAsB;oBACtB,cAAc;oBACd,+CAA+C;oBAC/C,2CAA2C;oBAC3C,mBAAmB;oBACnB,OAAO;iBACR;aACF;SACF,CAAC,CAAC;QAEH,IAAI,cAAc,KAAK,OAAO,EAAE,CAAC;YAC/B,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,kBAAQ,CAAC,MAAM,CAAC;gBAC/C;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,gBAAgB;oBACtB,OAAO,EACL,oFAAoF;iBACvF;aACF,CAAC,CAAC;YACH,QAAQ,GAAG,cAAc,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,cAAc,CAAC;QAC5B,CAAC;QAED,sBAAsB;QACtB,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,uBAAW,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAC7D,IAAI,EAAE,GAAG,MAAM,MAAM,mCAAuB,CAAC,MAAM,CAAC,IAAI,0BAA0B,EAAE;YACpF,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,2BAAe,CAAC,GAAG,CAAC,MAAM,CAAC;SACrC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,kBAAQ,CAAC,MAAM,CAAC;YAChD;gBACE,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,iBAAiB;gBACvB,OAAO,EAAE,wCAAwC;gBACjD,OAAO,EAAE,aAAa;gBACtB,QAAQ,EAAE,EAAE;aACb;SACF,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,eAAe,CAAC;QAEhC,qBAAqB;QACrB,MAAM,MAAM,GAAG;YACb,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,SAAS,EAAE,CAAC,QAAQ,CAAC;YACrB,KAAK,EAAE,EAAE;SACV,CAAC;QAEF,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAC;QACjE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,iBAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QAExD,gBAAM,CAAC,IAAI,CACT,IAAI,GAAG,eAAK,CAAC,KAAK,CAAC,6CAA6C,UAAU,EAAE,CAAC,GAAG,IAAI,CACrF,CAAC;QAEF,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,kBAAQ,CAAC,MAAM,CAAC;YAChD;gBACE,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,iBAAiB;gBACvB,OAAO,EAAE,mDAAmD;gBAC5D,OAAO,EAAE,IAAI;aACd;SACF,CAAC,CAAC;QAEH,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,cAAc;SACrB,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,IAAA,4BAAiB,EAAC;gBACtB,OAAO;gBACP,KAAK,EAAE,KAAK;gBACZ,KAAK,EAAE,IAAI;gBACX,aAAa,EAAE,MAAM;gBACrB,iBAAiB,EAAE,UAAU;aAC9B,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,gBAAM,CAAC,IAAI,CACT,IAAI;gBACF,eAAK,CAAC,IAAI,CACR,iDAAiD;oBAC/C,eAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAC3C,CACJ,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IAEL,cAAc;SACX,OAAO,CAAC,KAAK,CAAC;SACd,WAAW,CAAC,4BAA4B,CAAC;SACzC,MAAM,CAAC,qBAAqB,EAAE,4BAA4B,CAAC;SAC3D,MAAM,CAAC,YAAY,EAAE,eAAe,EAAE,KAAK,CAAC;SAC5C,MAAM,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;SAChD,MAAM,CAAC,YAAY,CAAC,CAAC;AAC1B,CAAC"}

package/dist/src/config.d.ts

@@ -0,0 +1,17 @@
+import { CommandLineOptions, TestSuite, UnifiedConfig } from './types';
+export declare function dereferenceConfig(rawConfig: UnifiedConfig): Promise<UnifiedConfig>;
+export declare function readConfig(configPath: string): Promise<UnifiedConfig>;
+export declare function maybeReadConfig(configPath: string): Promise<UnifiedConfig | undefined>;
+/**
+ * Reads multiple configuration files and combines them into a single UnifiedConfig.
+ *
+ * @param {string[]} configPaths - An array of paths to configuration files. Supports glob patterns.
+ * @returns {Promise<UnifiedConfig>} A promise that resolves to a unified configuration object.
+ */
+export declare function readConfigs(configPaths: string[]): Promise<UnifiedConfig>;
+export declare function resolveConfigs(cmdObj: Partial<CommandLineOptions>, defaultConfig: Partial<UnifiedConfig>): Promise<{
+    testSuite: TestSuite;
+    config: Partial<UnifiedConfig>;
+    basePath: string;
+}>;
+//# sourceMappingURL=config.d.ts.map

package/dist/src/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAgBA,OAAO,EACL,kBAAkB,EAKlB,SAAS,EACT,aAAa,EACd,MAAM,SAAS,CAAC;AAGjB,wBAAsB,iBAAiB,CAAC,SAAS,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,CAoGxF;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAe3E;AAED,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAK5F;AAED;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,CAkI/E;AAED,wBAAsB,cAAc,CAClC,MAAM,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACnC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,GACpC,OAAO,CAAC;IAAE,SAAS,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAsKrF"}