promptfoo 0.1.0

package/dist/logger.js

@@ -0,0 +1,38 @@
+import chalk from 'chalk';
+import winston from 'winston';
+const logLevels = {
+    error: 0,
+    warn: 1,
+    info: 2,
+    debug: 3,
+};
+const customFormatter = winston.format.printf(({ level, message, ...args }) => {
+    if (level === 'error') {
+        return chalk.red(message);
+    }
+    else if (level === 'warn') {
+        return chalk.yellow(message);
+    }
+    else if (level === 'info') {
+        return message;
+    }
+    else if (level === 'debug') {
+        return chalk.cyan(message);
+    }
+    throw new Error(`Invalid log level: ${level}`);
+});
+const logger = winston.createLogger({
+    levels: logLevels,
+    format: winston.format.combine(winston.format.simple(), customFormatter),
+    transports: [new winston.transports.Console()],
+});
+export function setLogLevel(level) {
+    if (logLevels.hasOwnProperty(level)) {
+        logger.transports[0].level = level;
+    }
+    else {
+        throw new Error(`Invalid log level: ${level}`);
+    }
+}
+export default logger;
+//# sourceMappingURL=logger.js.map

package/dist/logger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.js","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,MAAM,SAAS,GAAG;IAChB,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;CACT,CAAC;AAEF,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE;IAC5E,IAAI,KAAK,KAAK,OAAO,EAAE;QACrB,OAAO,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;KAC3B;SAAM,IAAI,KAAK,KAAK,MAAM,EAAE;QAC3B,OAAO,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;KAC9B;SAAM,IAAI,KAAK,KAAK,MAAM,EAAE;QAC3B,OAAO,OAAO,CAAC;KAChB;SAAM,IAAI,KAAK,KAAK,OAAO,EAAE;QAC5B,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;KAC5B;IACD,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,EAAE,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAClC,MAAM,EAAE,SAAS;IACjB,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,eAAe,CAAC;IACxE,UAAU,EAAE,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;CAC/C,CAAC,CAAC;AAEH,MAAM,UAAU,WAAW,CAAC,KAA6B;IACvD,IAAI,SAAS,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE;QACnC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;KACpC;SAAM;QACL,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,EAAE,CAAC,CAAC;KAChD;AACH,CAAC;AAED,eAAe,MAAM,CAAC"}

package/dist/main.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=main.d.ts.map

package/dist/main.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":""}

package/dist/main.js

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+import { readFileSync } from 'fs';
+import { parse } from 'path';
+import Table from 'cli-table3';
+import chalk from 'chalk';
+import { Command } from 'commander';
+import logger, { setLogLevel } from './logger.js';
+import { loadApiProvider } from './providers.js';
+import { evaluate } from './evaluator.js';
+import { readPrompts, readVars, writeOutput } from './util.js';
+const program = new Command();
+program
+    .command('eval')
+    .description('Evaluate prompts')
+    .requiredOption('-p, --prompt <paths...>', 'Paths to prompt files (.txt)')
+    .requiredOption('-r, --provider <name or path...>', 'One of: openai:chat, openai:completion, openai:<model name>, or path to custom API caller module')
+    .option('-o, --output <path>', 'Path to output file (csv, json, yaml, html)')
+    .option('-v, --vars <path>', 'Path to file with prompt variables (csv, json, yaml)')
+    .option('-c, --config <path>', 'Path to configuration file')
+    .option('-j, --max-concurrency <number>', 'Maximum number of concurrent API calls')
+    .option('--verbose', 'Show debug logs')
+    .action(async (cmdObj) => {
+    if (cmdObj.verbose) {
+        setLogLevel('debug');
+    }
+    const configPath = cmdObj.config;
+    let config = {};
+    if (configPath) {
+        const ext = parse(configPath).ext;
+        switch (ext) {
+            case '.json':
+                const content = readFileSync(configPath, 'utf-8');
+                config = JSON.parse(content);
+                break;
+            case '.js':
+                config = require(configPath);
+                break;
+            default:
+                throw new Error(`Unsupported configuration file format: ${ext}`);
+        }
+    }
+    let vars = [];
+    if (cmdObj.vars) {
+        vars = readVars(cmdObj.vars);
+    }
+    const providers = cmdObj.provider.map((p) => loadApiProvider(p));
+    const options = {
+        prompts: readPrompts(cmdObj.prompt),
+        vars,
+        providers,
+        showProgressBar: true,
+        maxConcurrency: cmdObj.maxConcurrency && cmdObj.maxConcurrency > 0 ? cmdObj.maxConcurrency : undefined,
+        ...config,
+    };
+    const summary = await evaluate(options);
+    if (cmdObj.output) {
+        logger.info(chalk.yellow(`Writing output to ${cmdObj.output}`));
+        writeOutput(cmdObj.output, summary);
+    }
+    else {
+        // Output table by default
+        const maxWidth = process.stdout.columns ? process.stdout.columns - 10 : 120;
+        const head = summary.table[0];
+        const table = new Table({
+            head,
+            colWidths: Array(head.length).fill(Math.floor(maxWidth / head.length)),
+            wordWrap: true,
+            wrapOnWordBoundary: true,
+            style: {
+                head: ['blue', 'bold'],
+            },
+        });
+        // Skip first row (header) and add the rest. Color the first column green if it's a success, red if it's a failure.
+        for (const row of summary.table.slice(1)) {
+            const color = row[0] === 'PASS' ? 'green' : row[0].startsWith('FAIL') ? 'red' : undefined;
+            table.push(row.map((col, i) => (i === 0 && color ? chalk[color](col) : col)));
+        }
+        logger.info('\n' + table.toString());
+    }
+    logger.info('Evaluation complete');
+    logger.info(chalk.green.bold(`Successes: ${summary.stats.successes}`));
+    logger.info(chalk.red.bold(`Failures: ${summary.stats.failures}`));
+    logger.info(`Token usage: Total ${summary.stats.tokenUsage.total} Prompt ${summary.stats.tokenUsage.prompt} Completion ${summary.stats.tokenUsage.completion}`);
+    logger.info('Done.');
+});
+program.parse(process.argv);
+if (!process.argv.slice(2).length) {
+    program.outputHelp();
+}
+//# sourceMappingURL=main.js.map

package/dist/main.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAE7B,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,MAAM,EAAE,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAI/D,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,kBAAkB,CAAC;KAC/B,cAAc,CAAC,yBAAyB,EAAE,8BAA8B,CAAC;KACzE,cAAc,CACb,kCAAkC,EAClC,kGAAkG,CACnG;KACA,MAAM,CAAC,qBAAqB,EAAE,6CAA6C,CAAC;KAC5E,MAAM,CAAC,mBAAmB,EAAE,sDAAsD,CAAC;KACnF,MAAM,CAAC,qBAAqB,EAAE,4BAA4B,CAAC;KAC3D,MAAM,CAAC,gCAAgC,EAAE,wCAAwC,CAAC;KAClF,MAAM,CAAC,WAAW,EAAE,iBAAiB,CAAC;KACtC,MAAM,CAAC,KAAK,EAAE,MAAoC,EAAE,EAAE;IACrD,IAAI,MAAM,CAAC,OAAO,EAAE;QAClB,WAAW,CAAC,OAAO,CAAC,CAAC;KACtB;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;IACjC,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,UAAU,EAAE;QACd,MAAM,GAAG,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC;QAClC,QAAQ,GAAG,EAAE;YACX,KAAK,OAAO;gBACV,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;gBAClD,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC7B,MAAM;YACR,KAAK,KAAK;gBACR,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;gBAC7B,MAAM;YACR;gBACE,MAAM,IAAI,KAAK,CAAC,0CAA0C,GAAG,EAAE,CAAC,CAAC;SACpE;KACF;IAED,IAAI,IAAI,GAAiB,EAAE,CAAC;IAC5B,IAAI,MAAM,CAAC,IAAI,EAAE;QACf,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;KAC9B;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,OAAO,GAAoB;QAC/B,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC;QACnC,IAAI;QACJ,SAAS;QACT,eAAe,EAAE,IAAI;QACrB,cAAc,EACZ,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS;QACxF,GAAG,MAAM;KACV,CAAC;IAEF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;IAExC,IAAI,MAAM,CAAC,MAAM,EAAE;QACjB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,qBAAqB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAChE,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACrC;SAAM;QACL,0BAA0B;QAC1B,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAC5E,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC;YACtB,IAAI;YACJ,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;YACtE,QAAQ,EAAE,IAAI;YACd,kBAAkB,EAAE,IAAI;YACxB,KAAK,EAAE;gBACL,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;aACvB;SACF,CAAC,CAAC;QACH,mHAAmH;QACnH,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YACxC,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YAC1F,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;SAC/E;QAED,MAAM,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;KACtC;IACD,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IACnC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,CAAC,IAAI,CACT,sBAAsB,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,WAAW,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,eAAe,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,EAAE,CACnJ,CAAC;IACF,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAE5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE;IACjC,OAAO,CAAC,UAAU,EAAE,CAAC;CACtB"}

package/dist/providers.d.ts

@@ -0,0 +1,21 @@
+import { ApiProvider, ProviderResponse } from './types.js';
+export declare class OpenAiGenericProvider implements ApiProvider {
+    modelName: string;
+    apiKey: string;
+    constructor(modelName: string, apiKey?: string);
+    id(): string;
+    toString(): string;
+    callApi(prompt: string): Promise<ProviderResponse>;
+}
+export declare class OpenAiCompletionProvider extends OpenAiGenericProvider {
+    static OPENAI_COMPLETION_MODELS: string[];
+    constructor(modelName: string, apiKey?: string);
+    callApi(prompt: string): Promise<ProviderResponse>;
+}
+export declare class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
+    static OPENAI_CHAT_MODELS: string[];
+    constructor(modelName: string, apiKey?: string);
+    callApi(prompt: string): Promise<ProviderResponse>;
+}
+export declare function loadApiProvider(providerPath: string): ApiProvider;
+//# sourceMappingURL=providers.d.ts.map

package/dist/providers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAG3D,qBAAa,qBAAsB,YAAW,WAAW;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;gBAEH,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAY9C,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAKZ,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAGzD;AAED,qBAAa,wBAAyB,SAAQ,qBAAqB;IACjE,MAAM,CAAC,wBAAwB,WAM7B;gBAEU,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAWxC,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA4BzD;AAED,qBAAa,4BAA6B,SAAQ,qBAAqB;IACrE,MAAM,CAAC,kBAAkB,WAOvB;gBAEU,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAWxC,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAoCzD;AAED,wBAAgB,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,WAAW,CAyBjE"}

package/dist/providers.js

@@ -0,0 +1,145 @@
+import fetch from 'node-fetch';
+import logger from './logger.js';
+export class OpenAiGenericProvider {
+    constructor(modelName, apiKey) {
+        this.modelName = modelName;
+        const key = apiKey || process.env.OPENAI_API_KEY;
+        if (!key) {
+            throw new Error('OpenAI API key is not set. Set OPENAI_API_KEY environment variable or pass it as an argument to the constructor.');
+        }
+        this.apiKey = key;
+    }
+    id() {
+        return `openai:${this.modelName}`;
+    }
+    toString() {
+        return `[OpenAI Provider ${this.modelName}]`;
+    }
+    // @ts-ignore: Prompt is not used in this implementation
+    async callApi(prompt) {
+        throw new Error('Not implemented');
+    }
+}
+class OpenAiCompletionProvider extends OpenAiGenericProvider {
+    constructor(modelName, apiKey) {
+        if (!OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.includes(modelName)) {
+            throw new Error(`Unknown OpenAI completion model name: ${modelName}. Use one of the following: ${OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.join(', ')}`);
+        }
+        super(modelName, apiKey);
+    }
+    async callApi(prompt) {
+        const body = {
+            model: this.modelName,
+            prompt,
+            max_tokens: process.env.OPENAI_MAX_TOKENS || 1024,
+            temperature: process.env.OPENAI_TEMPERATURE || 0,
+        };
+        logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
+        const response = await fetch('https://api.openai.com/v1/completions', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                Authorization: `Bearer ${this.apiKey}`,
+            },
+            body: JSON.stringify(body),
+        });
+        const data = (await response.json());
+        logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
+        return {
+            output: data.choices[0].text,
+            tokenUsage: {
+                total: data.usage.total_tokens,
+                prompt: data.usage.prompt_tokens,
+                completion: data.usage.completion_tokens,
+            },
+        };
+    }
+}
+OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS = [
+    'text-davinci-003',
+    'text-davinci-002',
+    'text-curie-001',
+    'text-babbage-001',
+    'text-ada-001',
+];
+export { OpenAiCompletionProvider };
+class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
+    constructor(modelName, apiKey) {
+        if (!OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS.includes(modelName)) {
+            throw new Error(`Unknown OpenAI completion model name: ${modelName}. Use one of the following: ${OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.join(', ')}`);
+        }
+        super(modelName, apiKey);
+    }
+    async callApi(prompt) {
+        let messages;
+        try {
+            // User can specify `messages` payload as JSON, or we'll just put the
+            // string prompt into a `messages` array.
+            messages = JSON.parse(prompt);
+        }
+        catch (e) {
+            messages = [{ role: 'user', content: prompt }];
+        }
+        const body = {
+            model: this.modelName,
+            messages: messages,
+            max_tokens: process.env.OPENAI_MAX_TOKENS || 1024,
+            temperature: process.env.OPENAI_MAX_TEMPERATURE || 0,
+        };
+        logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
+        const response = await fetch('https://api.openai.com/v1/chat/completions', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                Authorization: `Bearer ${this.apiKey}`,
+            },
+            body: JSON.stringify(body),
+        });
+        const data = (await response.json());
+        logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
+        return {
+            output: data.choices[0].message.content,
+            tokenUsage: {
+                total: data.usage.total_tokens,
+                prompt: data.usage.prompt_tokens,
+                completion: data.usage.completion_tokens,
+            },
+        };
+    }
+}
+OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS = [
+    'gpt-4',
+    'gpt-4-0314',
+    'gpt-4-32k',
+    'gpt-4-32k-0314',
+    'gpt-3.5-turbo',
+    'gpt-3.5-turbo-0301',
+];
+export { OpenAiChatCompletionProvider };
+export function loadApiProvider(providerPath) {
+    if (providerPath?.startsWith('openai:')) {
+        // Load OpenAI module
+        const options = providerPath.split(':');
+        const modelType = options[1];
+        const modelName = options[2];
+        if (modelType === 'chat') {
+            return new OpenAiChatCompletionProvider(modelName || 'gpt-3.5-turbo');
+        }
+        else if (modelType === 'completion') {
+            return new OpenAiCompletionProvider(modelName || 'text-davinci-003');
+        }
+        else if (OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS.includes(modelType)) {
+            return new OpenAiChatCompletionProvider(modelType);
+        }
+        else if (OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.includes(modelType)) {
+            return new OpenAiCompletionProvider(modelType);
+        }
+        else {
+            throw new Error(`Unknown OpenAI model type: ${modelType}. Use one of the following providers: openai:chat:<model name>, openai:completion:<model name>`);
+        }
+    }
+    // Load custom module
+    const CustomApiProvider = require(providerPath).default;
+    return new CustomApiProvider();
+}
+//# sourceMappingURL=providers.js.map

package/dist/providers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"providers.js","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,YAAY,CAAC;AAG/B,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,MAAM,OAAO,qBAAqB;IAIhC,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAE3B,MAAM,GAAG,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QACjD,IAAI,CAAC,GAAG,EAAE;YACR,MAAM,IAAI,KAAK,CACb,kHAAkH,CACnH,CAAC;SACH;QACD,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;IACpB,CAAC;IAED,EAAE;QACA,OAAO,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,CAAC;IAED,QAAQ;QACN,OAAO,oBAAoB,IAAI,CAAC,SAAS,GAAG,CAAC;IAC/C,CAAC;IAED,wDAAwD;IACxD,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;CACF;AAED,MAAa,wBAAyB,SAAQ,qBAAqB;IASjE,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,wBAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1E,MAAM,IAAI,KAAK,CACb,yCAAyC,SAAS,+BAA+B,wBAAwB,CAAC,wBAAwB,CAAC,IAAI,CACrI,IAAI,CACL,EAAE,CACJ,CAAC;SACH;QACD,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,MAAM;YACN,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;YACjD,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC;SACjD,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE;YACpE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aACvC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAmB,CAAC;QACvD,MAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;YAC5B,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;gBAC9B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;gBAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;aACzC;SACF,CAAC;IACJ,CAAC;;AA9CM,iDAAwB,GAAG;IAChC,kBAAkB;IAClB,kBAAkB;IAClB,gBAAgB;IAChB,kBAAkB;IAClB,cAAc;CACf,CAAC;SAPS,wBAAwB;AAkDrC,MAAa,4BAA6B,SAAQ,qBAAqB;IAUrE,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,4BAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YACxE,MAAM,IAAI,KAAK,CACb,yCAAyC,SAAS,+BAA+B,wBAAwB,CAAC,wBAAwB,CAAC,IAAI,CACrI,IAAI,CACL,EAAE,CACJ,CAAC;SACH;QACD,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,IAAI,QAA6C,CAAC;QAClD,IAAI;YACF,qEAAqE;YACrE,yCAAyC;YACzC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;SAC/B;QAAC,OAAO,CAAC,EAAE;YACV,QAAQ,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;SAChD;QACD,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,QAAQ,EAAE,QAAQ;YAClB,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;YACjD,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,CAAC;SACrD,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,4CAA4C,EAAE;YACzE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aACvC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAmB,CAAC;QACvD,MAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO;YACvC,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;gBAC9B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;gBAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;aACzC;SACF,CAAC;IACJ,CAAC;;AAvDM,+CAAkB,GAAG;IAC1B,OAAO;IACP,YAAY;IACZ,WAAW;IACX,gBAAgB;IAChB,eAAe;IACf,oBAAoB;CACrB,CAAC;SARS,4BAA4B;AA2DzC,MAAM,UAAU,eAAe,CAAC,YAAoB;IAClD,IAAI,YAAY,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE;QACvC,qBAAqB;QACrB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,4BAA4B,CAAC,SAAS,IAAI,eAAe,CAAC,CAAC;SACvE;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,wBAAwB,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;SACtE;aAAM,IAAI,4BAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC9E,OAAO,IAAI,4BAA4B,CAAC,SAAS,CAAC,CAAC;SACpD;aAAM,IAAI,wBAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAChF,OAAO,IAAI,wBAAwB,CAAC,SAAS,CAAC,CAAC;SAChD;aAAM;YACL,MAAM,IAAI,KAAK,CACb,8BAA8B,SAAS,gGAAgG,CACxI,CAAC;SACH;KACF;IAED,qBAAqB;IACrB,MAAM,iBAAiB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC;IACxD,OAAO,IAAI,iBAAiB,EAAE,CAAC;AACjC,CAAC"}

package/dist/tableOutput.html

@@ -0,0 +1,55 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width" />
+    <title>Table Output</title>
+    <style>
+      body {
+        font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Helvetica, Arial,
+          sans-serif;
+      }
+      table,
+      th,
+      td {
+        border: 1px solid black;
+        border-collapse: collapse;
+        text-align: left;
+        word-break: break-all;
+      }
+      th,
+      td {
+        padding: 5px;
+      }
+      /* If data-content is exactly "PASS", set font color to green */
+      tr > td[data-content="PASS"]:first-child {
+        color: green;
+        font-weight: bold;
+      }
+
+      /* If data-content starts with "FAIL", set font color to red */
+      tr > td[data-content^="FAIL"]:first-child {
+        color: red;
+        font-weight: bold;
+      }
+    </style>
+  </head>
+  <body>
+    <table>
+      <thead>
+        {% for header in table[0] %}
+        <th>{{ header }}</th>
+        {% endfor %}
+      </thead>
+      <tbody>
+        {% for row in table.slice(1) %}
+        <tr>
+          {% for cell in row %}
+          <td data-content="{{cell}}">{{ cell }}</td>
+          {% endfor %}
+        </tr>
+        {% endfor %}
+      </tbody>
+    </table>
+  </body>
+</html>

package/dist/types.d.ts

@@ -0,0 +1,55 @@
+export interface CommandLineOptions {
+    prompt: string[];
+    provider: string[];
+    output?: string;
+    vars?: string;
+    config?: string;
+    verbose?: boolean;
+    maxConcurrency?: number;
+}
+export interface ApiProvider {
+    id: () => string;
+    callApi: (prompt: string) => Promise<ProviderResponse>;
+}
+interface TokenUsage {
+    total: number;
+    prompt: number;
+    completion: number;
+}
+export interface ProviderResponse {
+    output: string;
+    tokenUsage?: TokenUsage;
+}
+export interface CsvRow {
+    [key: string]: string;
+}
+export type VarMapping = Record<string, string>;
+export interface EvaluateOptions {
+    providers: ApiProvider[];
+    prompts: string[];
+    vars?: VarMapping[];
+    maxConcurrency?: number;
+    showProgressBar?: boolean;
+}
+export interface Prompt {
+    raw: string;
+    display: string;
+}
+export interface EvaluateResult {
+    prompt: Prompt;
+    vars: Record<string, string>;
+    response?: ProviderResponse;
+    error?: string;
+    success: boolean;
+}
+export interface EvaluateSummary {
+    results: EvaluateResult[];
+    table: string[][];
+    stats: {
+        successes: number;
+        failures: number;
+        tokenUsage: TokenUsage;
+    };
+}
+export {};
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,MAAM,CAAC;IACjB,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACxD;AAED,UAAU,UAAU;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,WAAW,EAAE,CAAC;IACzB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,CAAC,EAAE,UAAU,EAAE,CAAC;IAEpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC;IAClB,KAAK,EAAE;QACL,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,UAAU,CAAC;KACxB,CAAC;CACH"}

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/dist/util.d.ts

@@ -0,0 +1,6 @@
+import { CsvRow } from './types.js';
+import type { EvaluateSummary } from './types.js';
+export declare function readPrompts(promptPaths: string[]): string[];
+export declare function readVars(varsPath: string): CsvRow[];
+export declare function writeOutput(outputPath: string, summary: EvaluateSummary): void;
+//# sourceMappingURL=util.d.ts.map

package/dist/util.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAMpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAYlD,wBAAgB,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAM3D;AAED,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAanD;AAED,wBAAgB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,GAAG,IAAI,CAoB9E"}

package/dist/util.js

@@ -0,0 +1,62 @@
+import * as fs from 'fs';
+import yaml from 'js-yaml';
+import nunjucks from 'nunjucks';
+import { parse as parsePath } from 'path';
+import { parse as parseCsv } from 'csv-parse/sync';
+import { stringify } from 'csv-stringify/sync';
+import { getDirectory } from './esm.js';
+const PROMPT_DELIMITER = '---';
+function parseJson(json) {
+    try {
+        return JSON.parse(json);
+    }
+    catch (err) {
+        return undefined;
+    }
+}
+export function readPrompts(promptPaths) {
+    let promptContents = promptPaths.map((path) => fs.readFileSync(path, 'utf-8'));
+    if (promptContents.length === 1) {
+        promptContents = promptContents[0].split(PROMPT_DELIMITER).map((p) => p.trim());
+    }
+    return promptContents;
+}
+export function readVars(varsPath) {
+    const fileExtension = parsePath(varsPath).ext.slice(1);
+    let rows = [];
+    if (fileExtension === 'csv') {
+        rows = parseCsv(fs.readFileSync(varsPath, 'utf-8'), { columns: true });
+    }
+    else if (fileExtension === 'json') {
+        rows = parseJson(fs.readFileSync(varsPath, 'utf-8'));
+    }
+    else if (fileExtension === 'yaml' || fileExtension === 'yml') {
+        rows = yaml.load(fs.readFileSync(varsPath, 'utf-8'));
+    }
+    return rows;
+}
+export function writeOutput(outputPath, summary) {
+    const outputExtension = outputPath.split('.').pop()?.toLowerCase();
+    if (outputExtension === 'csv' || outputExtension === 'txt') {
+        const csvOutput = stringify(summary.table);
+        fs.writeFileSync(outputPath, csvOutput);
+    }
+    else if (outputExtension === 'json') {
+        fs.writeFileSync(outputPath, JSON.stringify(summary, null, 2));
+    }
+    else if (outputExtension === 'yaml' || outputExtension === 'yml') {
+        fs.writeFileSync(outputPath, yaml.dump(summary));
+    }
+    else if (outputExtension === 'html') {
+        const template = fs.readFileSync(`${getDirectory()}/tableOutput.html`, 'utf-8');
+        const htmlOutput = nunjucks.renderString(template, {
+            table: summary.table,
+            results: summary.results,
+        });
+        fs.writeFileSync(outputPath, htmlOutput);
+    }
+    else {
+        throw new Error('Unsupported output file format. Use CSV, JSON, or YAML.');
+    }
+}
+//# sourceMappingURL=util.js.map

package/dist/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAEzB,OAAO,IAAI,MAAM,SAAS,CAAC;AAC3B,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAE1C,OAAO,EAAE,KAAK,IAAI,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE/C,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAIxC,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAE/B,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;KACzB;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,SAAS,CAAC;KAClB;AACH,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,WAAqB;IAC/C,IAAI,cAAc,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;IAC/E,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE;QAC/B,cAAc,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;KACjF;IACD,OAAO,cAAc,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,QAAgB;IACvC,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAa,EAAE,CAAC;IAExB,IAAI,aAAa,KAAK,KAAK,EAAE;QAC3B,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;KACxE;SAAM,IAAI,aAAa,KAAK,MAAM,EAAE;QACnC,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;KACtD;SAAM,IAAI,aAAa,KAAK,MAAM,IAAI,aAAa,KAAK,KAAK,EAAE;QAC9D,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAmB,CAAC;KACxE;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,UAAkB,EAAE,OAAwB;IACtE,MAAM,eAAe,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC;IAEnE,IAAI,eAAe,KAAK,KAAK,IAAI,eAAe,KAAK,KAAK,EAAE;QAC1D,MAAM,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC3C,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;KACzC;SAAM,IAAI,eAAe,KAAK,MAAM,EAAE;QACrC,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;KAChE;SAAM,IAAI,eAAe,KAAK,MAAM,IAAI,eAAe,KAAK,KAAK,EAAE;QAClE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;KAClD;SAAM,IAAI,eAAe,KAAK,MAAM,EAAE;QACrC,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,YAAY,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QAChF,MAAM,UAAU,GAAG,QAAQ,CAAC,YAAY,CAAC,QAAQ,EAAE;YACjD,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;KAC1C;SAAM;QACL,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;KAC5E;AACH,CAAC"}

package/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "promptfoo",
+  "author": "Ian Webster",
+  "version": "0.1.0",
+  "license": "MIT",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "typings": "dist/index.d.ts",
+  "files": [
+    "dist",
+    "src"
+  ],
+  "engines": {
+    "node": ">=12"
+  },
+  "bin": {
+    "promptfoo": "dist/main.js"
+  },
+  "scripts": {
+    "local": "ts-node --esm src/main.ts",
+    "build": "tsc && cp src/*.html dist/",
+    "build:watch": "tsc --watch",
+    "prepare": "tsc",
+    "test": "jest",
+    "test:watch": "jest --watch",
+    "format": "prettier -w ."
+  },
+  "devDependencies": {
+    "@types/async": "^3.2.20",
+    "@types/cli-progress": "^3.11.0",
+    "@types/jest": "^29.5.1",
+    "@types/js-yaml": "^4.0.5",
+    "@types/nunjucks": "^3.2.2",
+    "babel-jest": "^29.5.0",
+    "jest-watch-typeahead": "^2.2.2",
+    "prettier": "^2.8.8",
+    "ts-jest": "^29.1.0",
+    "ts-node": "^10.9.1",
+    "typescript": "^5.0.4"
+  },
+  "dependencies": {
+    "async": "^3.2.4",
+    "chalk": "^4.1.2",
+    "cli-progress": "^3.12.0",
+    "cli-table3": "^0.6.3",
+    "commander": "^10.0.1",
+    "csv-parse": "^5.3.8",
+    "csv-stringify": "^6.3.2",
+    "js-yaml": "^4.1.0",
+    "node-fetch": "^3.3.1",
+    "nunjucks": "^3.2.4",
+    "winston": "^3.8.2"
+  }
+}

package/src/__mocks__/esm.ts

@@ -0,0 +1,3 @@
+export function getDirectory() {
+  return '/test/dir';
+}

package/src/esm.ts

@@ -0,0 +1,10 @@
+// esm-specific crap that needs to get mocked out in tests
+
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+export function getDirectory(): string {
+  // @ts-ignore: Jest chokes on this
+  const __filename = fileURLToPath(import.meta.url);
+  return path.dirname(__filename);
+}