promptfoo 0.69.1 → 0.70.0

package/dist/src/commands/redteam.js

@@ -0,0 +1,191 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.doRunRedteam = doRunRedteam;
+exports.redteamCommand = redteamCommand;
+const chalk_1 = __importDefault(require("chalk"));
+const fs = __importStar(require("fs"));
+const inquirer_1 = __importDefault(require("inquirer"));
+const js_yaml_1 = __importDefault(require("js-yaml"));
+const path = __importStar(require("path"));
+const logger_1 = __importDefault(require("../logger"));
+const constants_1 = require("../redteam/constants");
+const telemetry_1 = __importDefault(require("../telemetry"));
+const generate_1 = require("./generate");
+async function doRunRedteam(cmdObj) { }
+function redteamCommand(program) {
+    const redteamCommand = program.command('redteam').description('Red team LLM applications');
+    redteamCommand
+        .command('init [directory]')
+        .description('Initialize red teaming project')
+        .action(async (directory) => {
+        telemetry_1.default.maybeShowNotice();
+        telemetry_1.default.record('command_used', {
+            name: 'redteam init - started',
+        });
+        await telemetry_1.default.send();
+        let projectDir = directory;
+        if (!projectDir) {
+            const { chosenDir } = await inquirer_1.default.prompt([
+                {
+                    type: 'input',
+                    name: 'chosenDir',
+                    message: 'Where do you want to create the project?',
+                    default: '.',
+                },
+            ]);
+            projectDir = chosenDir || '.';
+        }
+        if (projectDir !== '.' && !fs.existsSync(projectDir)) {
+            fs.mkdirSync(projectDir, { recursive: true });
+        }
+        // Question 2: Prompt
+        const { promptChoice } = await inquirer_1.default.prompt([
+            {
+                type: 'list',
+                name: 'promptChoice',
+                message: 'How would you like to specify the prompt?',
+                choices: [
+                    { name: 'Enter a prompt', value: 'enter' },
+                    { name: 'Reference a prompt file', value: 'file' },
+                ],
+            },
+        ]);
+        let prompt;
+        if (promptChoice === 'enter') {
+            const { enteredPrompt } = await inquirer_1.default.prompt([
+                {
+                    type: 'editor',
+                    name: 'enteredPrompt',
+                    message: 'Enter your prompt:',
+                    default: 'You are a helpful concise assistant.\n\nUser query: {{query}}\n\n(NOTE: your prompt must include "{{query}}" as a placeholder for user input)',
+                },
+            ]);
+            prompt = enteredPrompt;
+        }
+        else {
+            const { promptFile } = await inquirer_1.default.prompt([
+                {
+                    type: 'input',
+                    name: 'promptFile',
+                    message: 'Enter the path to your prompt file (text or JSON):',
+                },
+            ]);
+            prompt = `file://${promptFile}`;
+        }
+        // Question 3: Provider
+        let provider;
+        const { providerChoice } = await inquirer_1.default.prompt([
+            {
+                type: 'list',
+                name: 'providerChoice',
+                message: 'Choose a provider:',
+                choices: [
+                    'openai:gpt-3.5-turbo',
+                    'openai:gpt-4',
+                    'anthropic:messages:claude-3-5-sonnet-20240620',
+                    'anthropic:messages:claude-3-opus-20240307',
+                    'vertex:gemini-pro',
+                    'Other',
+                ],
+            },
+        ]);
+        if (providerChoice === 'Other') {
+            const { customProvider } = await inquirer_1.default.prompt([
+                {
+                    type: 'input',
+                    name: 'customProvider',
+                    message: 'Enter the provider ID (see https://www.promptfoo.dev/docs/providers/ for options):',
+                },
+            ]);
+            provider = customProvider;
+        }
+        else {
+            provider = providerChoice;
+        }
+        // Question 4: Plugins
+        const pluginChoices = Array.from(constants_1.ALL_PLUGINS).map((plugin) => ({
+            name: `${plugin} - ${constants_1.subCategoryDescriptions[plugin] || 'No description available'}`,
+            value: plugin,
+            checked: constants_1.DEFAULT_PLUGINS.has(plugin),
+        }));
+        const { selectedPlugins } = await inquirer_1.default.prompt([
+            {
+                type: 'checkbox',
+                name: 'selectedPlugins',
+                message: 'Select the plugins you want to enable:',
+                choices: pluginChoices,
+                pageSize: 20,
+            },
+        ]);
+        const plugins = selectedPlugins;
+        // Create config file
+        const config = {
+            prompts: [prompt],
+            providers: [provider],
+            tests: [],
+        };
+        const configPath = path.join(projectDir, 'promptfooconfig.yaml');
+        fs.writeFileSync(configPath, js_yaml_1.default.dump(config), 'utf8');
+        logger_1.default.info('\n' + chalk_1.default.green(`Created red teaming configuration file at ${configPath}`) + '\n');
+        const { readyToGenerate } = await inquirer_1.default.prompt([
+            {
+                type: 'confirm',
+                name: 'readyToGenerate',
+                message: 'Are you ready to generate adversarial test cases?',
+                default: true,
+            },
+        ]);
+        telemetry_1.default.record('command_used', {
+            name: 'redteam init',
+        });
+        await telemetry_1.default.send();
+        if (readyToGenerate) {
+            await (0, generate_1.doGenerateRedteam)({
+                plugins,
+                cache: false,
+                write: true,
+                defaultConfig: config,
+                defaultConfigPath: configPath,
+            });
+        }
+        else {
+            logger_1.default.info('\n' +
+                chalk_1.default.blue('To generate test cases later, use the command: ' +
+                    chalk_1.default.bold('promptfoo generate redteam')));
+        }
+    });
+    redteamCommand
+        .command('run')
+        .description('Run red teaming evaluation')
+        .option('-c, --config <path>', 'Path to configuration file')
+        .option('--no-cache', 'Disable cache', false)
+        .option('--env-file <path>', 'Path to .env file')
+        .action(doRunRedteam);
+}
+//# sourceMappingURL=redteam.js.map

package/dist/src/commands/redteam.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"redteam.js","sourceRoot":"","sources":["../../../src/commands/redteam.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiBA,oCAAgE;AAEhE,wCA0KC;AA7LD,kDAA0B;AAE1B,uCAAyB;AACzB,wDAAgC;AAChC,sDAA2B;AAC3B,2CAA6B;AAC7B,uDAA+B;AAC/B,oDAA6F;AAC7F,6DAAqC;AACrC,yCAA+C;AAQxC,KAAK,UAAU,YAAY,CAAC,MAAyB,IAAG,CAAC;AAEhE,SAAgB,cAAc,CAAC,OAAgB;IAC7C,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,WAAW,CAAC,2BAA2B,CAAC,CAAC;IAE3F,cAAc;SACX,OAAO,CAAC,kBAAkB,CAAC;SAC3B,WAAW,CAAC,gCAAgC,CAAC;SAC7C,MAAM,CAAC,KAAK,EAAE,SAA6B,EAAE,EAAE;QAC9C,mBAAS,CAAC,eAAe,EAAE,CAAC;QAC5B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,wBAAwB;SAC/B,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,IAAI,UAAU,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,kBAAQ,CAAC,MAAM,CAAC;gBAC1C;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,0CAA0C;oBACnD,OAAO,EAAE,GAAG;iBACb;aACF,CAAC,CAAC;YACH,UAAU,GAAI,SAAgC,IAAI,GAAG,CAAC;QACxD,CAAC;QACD,IAAI,UAAU,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YACrD,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,CAAC;QAED,qBAAqB;QACrB,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,kBAAQ,CAAC,MAAM,CAAC;YAC7C;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,cAAc;gBACpB,OAAO,EAAE,2CAA2C;gBACpD,OAAO,EAAE;oBACP,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,OAAO,EAAE;oBAC1C,EAAE,IAAI,EAAE,yBAAyB,EAAE,KAAK,EAAE,MAAM,EAAE;iBACnD;aACF;SACF,CAAC,CAAC;QAEH,IAAI,MAAc,CAAC;QACnB,IAAI,YAAY,KAAK,OAAO,EAAE,CAAC;YAC7B,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,kBAAQ,CAAC,MAAM,CAAC;gBAC9C;oBACE,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,eAAe;oBACrB,OAAO,EAAE,oBAAoB;oBAC7B,OAAO,EACL,+IAA+I;iBAClJ;aACF,CAAC,CAAC;YACH,MAAM,GAAG,aAAa,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,kBAAQ,CAAC,MAAM,CAAC;gBAC3C;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,YAAY;oBAClB,OAAO,EAAE,oDAAoD;iBAC9D;aACF,CAAC,CAAC;YACH,MAAM,GAAG,UAAU,UAAU,EAAE,CAAC;QAClC,CAAC;QAED,uBAAuB;QACvB,IAAI,QAAgB,CAAC;QACrB,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,kBAAQ,CAAC,MAAM,CAAC;YAC/C;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,gBAAgB;gBACtB,OAAO,EAAE,oBAAoB;gBAC7B,OAAO,EAAE;oBACP,sBAAsB;oBACtB,cAAc;oBACd,+CAA+C;oBAC/C,2CAA2C;oBAC3C,mBAAmB;oBACnB,OAAO;iBACR;aACF;SACF,CAAC,CAAC;QAEH,IAAI,cAAc,KAAK,OAAO,EAAE,CAAC;YAC/B,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,kBAAQ,CAAC,MAAM,CAAC;gBAC/C;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,gBAAgB;oBACtB,OAAO,EACL,oFAAoF;iBACvF;aACF,CAAC,CAAC;YACH,QAAQ,GAAG,cAAc,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,cAAc,CAAC;QAC5B,CAAC;QAED,sBAAsB;QACtB,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,uBAAW,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAC7D,IAAI,EAAE,GAAG,MAAM,MAAM,mCAAuB,CAAC,MAAM,CAAC,IAAI,0BAA0B,EAAE;YACpF,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,2BAAe,CAAC,GAAG,CAAC,MAAM,CAAC;SACrC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,kBAAQ,CAAC,MAAM,CAAC;YAChD;gBACE,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,iBAAiB;gBACvB,OAAO,EAAE,wCAAwC;gBACjD,OAAO,EAAE,aAAa;gBACtB,QAAQ,EAAE,EAAE;aACb;SACF,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,eAAe,CAAC;QAEhC,qBAAqB;QACrB,MAAM,MAAM,GAAG;YACb,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,SAAS,EAAE,CAAC,QAAQ,CAAC;YACrB,KAAK,EAAE,EAAE;SACV,CAAC;QAEF,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAC;QACjE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,iBAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QAExD,gBAAM,CAAC,IAAI,CACT,IAAI,GAAG,eAAK,CAAC,KAAK,CAAC,6CAA6C,UAAU,EAAE,CAAC,GAAG,IAAI,CACrF,CAAC;QAEF,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,kBAAQ,CAAC,MAAM,CAAC;YAChD;gBACE,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,iBAAiB;gBACvB,OAAO,EAAE,mDAAmD;gBAC5D,OAAO,EAAE,IAAI;aACd;SACF,CAAC,CAAC;QAEH,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,cAAc;SACrB,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,IAAA,4BAAiB,EAAC;gBACtB,OAAO;gBACP,KAAK,EAAE,KAAK;gBACZ,KAAK,EAAE,IAAI;gBACX,aAAa,EAAE,MAAM;gBACrB,iBAAiB,EAAE,UAAU;aAC9B,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,gBAAM,CAAC,IAAI,CACT,IAAI;gBACF,eAAK,CAAC,IAAI,CACR,iDAAiD;oBAC/C,eAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAC3C,CACJ,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IAEL,cAAc;SACX,OAAO,CAAC,KAAK,CAAC;SACd,WAAW,CAAC,4BAA4B,CAAC;SACzC,MAAM,CAAC,qBAAqB,EAAE,4BAA4B,CAAC;SAC3D,MAAM,CAAC,YAAY,EAAE,eAAe,EAAE,KAAK,CAAC;SAC5C,MAAM,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;SAChD,MAAM,CAAC,YAAY,CAAC,CAAC;AAC1B,CAAC"}

package/dist/src/config.d.ts

@@ -0,0 +1,17 @@
+import { CommandLineOptions, TestSuite, UnifiedConfig } from './types';
+export declare function dereferenceConfig(rawConfig: UnifiedConfig): Promise<UnifiedConfig>;
+export declare function readConfig(configPath: string): Promise<UnifiedConfig>;
+export declare function maybeReadConfig(configPath: string): Promise<UnifiedConfig | undefined>;
+/**
+ * Reads multiple configuration files and combines them into a single UnifiedConfig.
+ *
+ * @param {string[]} configPaths - An array of paths to configuration files. Supports glob patterns.
+ * @returns {Promise<UnifiedConfig>} A promise that resolves to a unified configuration object.
+ */
+export declare function readConfigs(configPaths: string[]): Promise<UnifiedConfig>;
+export declare function resolveConfigs(cmdObj: Partial<CommandLineOptions>, defaultConfig: Partial<UnifiedConfig>): Promise<{
+    testSuite: TestSuite;
+    config: Partial<UnifiedConfig>;
+    basePath: string;
+}>;
+//# sourceMappingURL=config.d.ts.map

package/dist/src/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAgBA,OAAO,EACL,kBAAkB,EAKlB,SAAS,EACT,aAAa,EACd,MAAM,SAAS,CAAC;AAGjB,wBAAsB,iBAAiB,CAAC,SAAS,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,CAoGxF;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAe3E;AAED,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAK5F;AAED;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,CAkI/E;AAED,wBAAsB,cAAc,CAClC,MAAM,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACnC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,GACpC,OAAO,CAAC;IAAE,SAAS,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAsKrF"}

package/dist/src/config.js

@@ -0,0 +1,424 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.dereferenceConfig = dereferenceConfig;
+exports.readConfig = readConfig;
+exports.maybeReadConfig = maybeReadConfig;
+exports.readConfigs = readConfigs;
+exports.resolveConfigs = resolveConfigs;
+const json_schema_ref_parser_1 = __importDefault(require("@apidevtools/json-schema-ref-parser"));
+const chalk_1 = __importDefault(require("chalk"));
+const dedent_1 = __importDefault(require("dedent"));
+const fs = __importStar(require("fs"));
+const glob_1 = require("glob");
+const js_yaml_1 = __importDefault(require("js-yaml"));
+const path = __importStar(require("path"));
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+const assertions_1 = require("./assertions");
+const validateAssertions_1 = require("./assertions/validateAssertions");
+const filterTests_1 = require("./commands/eval/filterTests");
+const esm_1 = require("./esm");
+const logger_1 = __importDefault(require("./logger"));
+const prompts_1 = require("./prompts");
+const providers_1 = require("./providers");
+const testCases_1 = require("./testCases");
+const types_1 = require("./types");
+const util_1 = require("./util");
+async function dereferenceConfig(rawConfig) {
+    if (process.env.PROMPTFOO_DISABLE_REF_PARSER) {
+        return rawConfig;
+    }
+    // Track and delete tools[i].function for each tool, preserving the rest of the properties
+    // https://github.com/promptfoo/promptfoo/issues/364
+    // Remove parameters from functions and tools to prevent dereferencing
+    const extractFunctionParameters = (functions) => {
+        return functions.map((func) => {
+            const { parameters } = func;
+            delete func.parameters;
+            return { parameters };
+        });
+    };
+    const extractToolParameters = (tools) => {
+        return tools.map((tool) => {
+            const { parameters } = tool.function || {};
+            if (tool.function?.parameters) {
+                delete tool.function.parameters;
+            }
+            return { parameters };
+        });
+    };
+    // Restore parameters to functions and tools after dereferencing
+    const restoreFunctionParameters = (functions, parametersList) => {
+        functions.forEach((func, index) => {
+            if (parametersList[index]?.parameters) {
+                func.parameters = parametersList[index].parameters;
+            }
+        });
+    };
+    const restoreToolParameters = (tools, parametersList) => {
+        tools.forEach((tool, index) => {
+            if (parametersList[index]?.parameters) {
+                tool.function = tool.function || {};
+                tool.function.parameters = parametersList[index].parameters;
+            }
+        });
+    };
+    const functionsParametersList = [];
+    const toolsParametersList = [];
+    if (Array.isArray(rawConfig.providers)) {
+        rawConfig.providers.forEach((provider, providerIndex) => {
+            if (typeof provider === 'string')
+                return;
+            if (typeof provider === 'function')
+                return;
+            if (!provider.config) {
+                // Handle when provider is a map
+                provider = Object.values(provider)[0];
+            }
+            if (provider.config?.functions) {
+                functionsParametersList[providerIndex] = extractFunctionParameters(provider.config.functions);
+            }
+            if (provider.config?.tools) {
+                toolsParametersList[providerIndex] = extractToolParameters(provider.config.tools);
+            }
+        });
+    }
+    // Dereference JSON
+    const config = (await json_schema_ref_parser_1.default.dereference(rawConfig));
+    // Restore functions and tools parameters
+    if (Array.isArray(config.providers)) {
+        config.providers.forEach((provider, index) => {
+            if (typeof provider === 'string')
+                return;
+            if (typeof provider === 'function')
+                return;
+            if (!provider.config) {
+                // Handle when provider is a map
+                provider = Object.values(provider)[0];
+            }
+            if (functionsParametersList[index]) {
+                provider.config.functions = provider.config.functions || [];
+                restoreFunctionParameters(provider.config.functions, functionsParametersList[index]);
+            }
+            if (toolsParametersList[index]) {
+                provider.config.tools = provider.config.tools || [];
+                restoreToolParameters(provider.config.tools, toolsParametersList[index]);
+            }
+        });
+    }
+    return config;
+}
+async function readConfig(configPath) {
+    const ext = path.parse(configPath).ext;
+    switch (ext) {
+        case '.json':
+        case '.yaml':
+        case '.yml':
+            const rawConfig = js_yaml_1.default.load(fs.readFileSync(configPath, 'utf-8'));
+            return dereferenceConfig(rawConfig);
+        case '.js':
+        case '.cjs':
+        case '.mjs':
+            return (await (0, esm_1.importModule)(configPath));
+        default:
+            throw new Error(`Unsupported configuration file format: ${ext}`);
+    }
+}
+async function maybeReadConfig(configPath) {
+    if (!fs.existsSync(configPath)) {
+        return undefined;
+    }
+    return readConfig(configPath);
+}
+/**
+ * Reads multiple configuration files and combines them into a single UnifiedConfig.
+ *
+ * @param {string[]} configPaths - An array of paths to configuration files. Supports glob patterns.
+ * @returns {Promise<UnifiedConfig>} A promise that resolves to a unified configuration object.
+ */
+async function readConfigs(configPaths) {
+    const configs = [];
+    for (const configPath of configPaths) {
+        const globPaths = (0, glob_1.globSync)(configPath, {
+            windowsPathsNoEscape: true,
+        });
+        if (globPaths.length === 0) {
+            throw new Error(`No configuration file found at ${configPath}`);
+        }
+        for (const globPath of globPaths) {
+            const config = await readConfig(globPath);
+            configs.push(config);
+        }
+    }
+    const providers = [];
+    const seenProviders = new Set();
+    configs.forEach((config) => {
+        (0, tiny_invariant_1.default)(typeof config.providers !== 'function', 'Providers cannot be a function for multiple configs');
+        if (typeof config.providers === 'string') {
+            if (!seenProviders.has(config.providers)) {
+                providers.push(config.providers);
+                seenProviders.add(config.providers);
+            }
+        }
+        else if (Array.isArray(config.providers)) {
+            config.providers.forEach((provider) => {
+                if (!seenProviders.has(JSON.stringify(provider))) {
+                    providers.push(provider);
+                    seenProviders.add(JSON.stringify(provider));
+                }
+            });
+        }
+    });
+    const tests = [];
+    for (const config of configs) {
+        if (typeof config.tests === 'string') {
+            const newTests = await (0, testCases_1.readTests)(config.tests, path.dirname(configPaths[0]));
+            tests.push(...newTests);
+        }
+        else if (Array.isArray(config.tests)) {
+            tests.push(...config.tests);
+        }
+    }
+    const configsAreStringOrArray = configs.every((config) => typeof config.prompts === 'string' || Array.isArray(config.prompts));
+    const configsAreObjects = configs.every((config) => typeof config.prompts === 'object');
+    let prompts = configsAreStringOrArray ? [] : {};
+    const makeAbsolute = (configPath, relativePath) => {
+        if (typeof relativePath === 'string') {
+            if (relativePath.startsWith('file://')) {
+                relativePath =
+                    'file://' + path.resolve(path.dirname(configPath), relativePath.slice('file://'.length));
+            }
+            return relativePath;
+        }
+        else if (typeof relativePath === 'object' && relativePath.id) {
+            if (relativePath.id.startsWith('file://')) {
+                relativePath.id =
+                    'file://' +
+                        path.resolve(path.dirname(configPath), relativePath.id.slice('file://'.length));
+            }
+            return relativePath;
+        }
+        else {
+            throw new Error('Invalid prompt object');
+        }
+    };
+    const seenPrompts = new Set();
+    const addSeenPrompt = (prompt) => {
+        if (typeof prompt === 'string') {
+            seenPrompts.add(prompt);
+        }
+        else if (typeof prompt === 'object' && prompt.id) {
+            seenPrompts.add(prompt);
+        }
+        else {
+            throw new Error('Invalid prompt object');
+        }
+    };
+    configs.forEach((config, idx) => {
+        if (typeof config.prompts === 'string') {
+            (0, tiny_invariant_1.default)(Array.isArray(prompts), 'Cannot mix string and map-type prompts');
+            const absolutePrompt = makeAbsolute(configPaths[idx], config.prompts);
+            addSeenPrompt(absolutePrompt);
+        }
+        else if (Array.isArray(config.prompts)) {
+            (0, tiny_invariant_1.default)(Array.isArray(prompts), 'Cannot mix configs with map and array-type prompts');
+            config.prompts
+                .map((prompt) => makeAbsolute(configPaths[idx], prompt))
+                .forEach((prompt) => addSeenPrompt(prompt));
+        }
+        else {
+            // Object format such as { 'prompts/prompt1.txt': 'foo', 'prompts/prompt2.txt': 'bar' }
+            (0, tiny_invariant_1.default)(typeof prompts === 'object', 'Cannot mix configs with map and array-type prompts');
+            prompts = { ...prompts, ...config.prompts };
+        }
+    });
+    if (Array.isArray(prompts)) {
+        prompts.push(...Array.from(seenPrompts));
+    }
+    // Combine all configs into a single UnifiedConfig
+    const combinedConfig = {
+        description: configs.map((config) => config.description).join(', '),
+        providers,
+        prompts,
+        tests,
+        scenarios: configs.flatMap((config) => config.scenarios || []),
+        defaultTest: configs.reduce((prev, curr) => {
+            return {
+                ...prev,
+                ...curr.defaultTest,
+                vars: { ...prev?.vars, ...curr.defaultTest?.vars },
+                assert: [...(prev?.assert || []), ...(curr.defaultTest?.assert || [])],
+                options: { ...prev?.options, ...curr.defaultTest?.options },
+            };
+        }, {}),
+        nunjucksFilters: configs.reduce((prev, curr) => ({ ...prev, ...curr.nunjucksFilters }), {}),
+        env: configs.reduce((prev, curr) => ({ ...prev, ...curr.env }), {}),
+        evaluateOptions: configs.reduce((prev, curr) => ({ ...prev, ...curr.evaluateOptions }), {}),
+        commandLineOptions: configs.reduce((prev, curr) => ({ ...prev, ...curr.commandLineOptions }), {}),
+        metadata: configs.reduce((prev, curr) => ({ ...prev, ...curr.metadata }), {}),
+        sharing: !configs.some((config) => config.sharing === false),
+    };
+    return combinedConfig;
+}
+async function resolveConfigs(cmdObj, defaultConfig) {
+    // Config parsing
+    let fileConfig = {};
+    const configPaths = cmdObj.config;
+    if (configPaths) {
+        fileConfig = await readConfigs(configPaths);
+    }
+    // Standalone assertion mode
+    if (cmdObj.assertions) {
+        if (!cmdObj.modelOutputs) {
+            logger_1.default.error(chalk_1.default.red('You must provide --model-outputs when using --assertions'));
+            process.exit(1);
+        }
+        const modelOutputs = JSON.parse(fs.readFileSync(path.join(process.cwd(), cmdObj.modelOutputs), 'utf8'));
+        const assertions = await (0, assertions_1.readAssertions)(cmdObj.assertions);
+        fileConfig.prompts = ['{{output}}'];
+        fileConfig.providers = ['echo'];
+        fileConfig.tests = modelOutputs.map((output) => {
+            if (typeof output === 'string') {
+                return {
+                    vars: {
+                        output,
+                    },
+                    assert: assertions,
+                };
+            }
+            return {
+                vars: {
+                    output: output.output,
+                    ...(output.tags === undefined ? {} : { tags: output.tags.join(', ') }),
+                },
+                assert: assertions,
+            };
+        });
+    }
+    // Use basepath in cases where path was supplied in the config file
+    const basePath = configPaths ? path.dirname(configPaths[0]) : '';
+    const defaultTestRaw = fileConfig.defaultTest || defaultConfig.defaultTest;
+    const config = {
+        description: fileConfig.description || defaultConfig.description,
+        prompts: cmdObj.prompts || fileConfig.prompts || defaultConfig.prompts || [],
+        providers: cmdObj.providers || fileConfig.providers || defaultConfig.providers || [],
+        tests: cmdObj.tests || cmdObj.vars || fileConfig.tests || defaultConfig.tests || [],
+        scenarios: fileConfig.scenarios || defaultConfig.scenarios,
+        env: fileConfig.env || defaultConfig.env,
+        sharing: process.env.PROMPTFOO_DISABLE_SHARING === '1'
+            ? false
+            : fileConfig.sharing ?? defaultConfig.sharing ?? true,
+        defaultTest: defaultTestRaw ? await (0, testCases_1.readTest)(defaultTestRaw, basePath) : undefined,
+        derivedMetrics: fileConfig.derivedMetrics || defaultConfig.derivedMetrics,
+        outputPath: cmdObj.output || fileConfig.outputPath || defaultConfig.outputPath,
+        metadata: fileConfig.metadata || defaultConfig.metadata,
+    };
+    // Validation
+    if (!config.prompts || config.prompts.length === 0) {
+        logger_1.default.error(chalk_1.default.red('You must provide at least 1 prompt'));
+        process.exit(1);
+    }
+    if (!config.providers || config.providers.length === 0) {
+        logger_1.default.error(chalk_1.default.red('You must specify at least 1 provider (for example, openai:gpt-4o)'));
+        process.exit(1);
+    }
+    (0, tiny_invariant_1.default)(Array.isArray(config.providers), 'providers must be an array');
+    config.providers.forEach((provider) => {
+        const result = types_1.ProviderSchema.safeParse(provider);
+        if (!result.success) {
+            const errors = result.error.errors
+                .map((err) => {
+                return `- ${err.message}`;
+            })
+                .join('\n');
+            const providerString = typeof provider === 'string' ? provider : JSON.stringify(provider);
+            logger_1.default.warn(chalk_1.default.yellow((0, dedent_1.default) `
+              Provider: ${providerString} encountered errors during schema validation:
+
+                ${errors}
+
+              Please double check your configuration.` + '\n'));
+        }
+    });
+    // Parse prompts, providers, and tests
+    const parsedPrompts = await (0, prompts_1.readPrompts)(config.prompts, cmdObj.prompts ? undefined : basePath);
+    const parsedProviders = await (0, providers_1.loadApiProviders)(config.providers, {
+        env: config.env,
+        basePath,
+    });
+    const parsedTests = await (0, testCases_1.readTests)(config.tests || [], cmdObj.tests ? undefined : basePath);
+    // Parse testCases for each scenario
+    if (fileConfig.scenarios) {
+        for (const scenario of fileConfig.scenarios) {
+            const parsedScenarioTests = await (0, testCases_1.readTests)(scenario.tests, cmdObj.tests ? undefined : basePath);
+            scenario.tests = parsedScenarioTests;
+            const filteredTests = await (0, filterTests_1.filterTests)({
+                ...scenario,
+                providers: parsedProviders,
+                prompts: parsedPrompts,
+            }, {
+                firstN: cmdObj.filterFirstN,
+                pattern: cmdObj.filterPattern,
+                failing: cmdObj.filterFailing,
+            });
+            (0, tiny_invariant_1.default)(filteredTests, 'filteredTests are undefined');
+            scenario.tests = filteredTests;
+        }
+    }
+    const parsedProviderPromptMap = (0, prompts_1.readProviderPromptMap)(config, parsedPrompts);
+    if (parsedPrompts.length === 0) {
+        logger_1.default.error(chalk_1.default.red('No prompts found'));
+        process.exit(1);
+    }
+    const defaultTest = {
+        options: {
+            prefix: cmdObj.promptPrefix,
+            suffix: cmdObj.promptSuffix,
+            provider: cmdObj.grader,
+            // rubricPrompt
+            ...(config.defaultTest?.options || {}),
+        },
+        ...config.defaultTest,
+    };
+    const testSuite = {
+        description: config.description,
+        prompts: parsedPrompts,
+        providers: parsedProviders,
+        providerPromptMap: parsedProviderPromptMap,
+        tests: parsedTests,
+        scenarios: config.scenarios,
+        defaultTest,
+        derivedMetrics: config.derivedMetrics,
+        nunjucksFilters: await (0, util_1.readFilters)(fileConfig.nunjucksFilters || defaultConfig.nunjucksFilters || {}),
+    };
+    if (testSuite.tests) {
+        (0, validateAssertions_1.validateAssertions)(testSuite.tests);
+    }
+    return { config, testSuite, basePath };
+}
+//# sourceMappingURL=config.js.map