promptfoo 0.20.0 → 0.21.0

package/src/main.ts

@@ -1,442 +0,0 @@
-#!/usr/bin/env node
-import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs';
-import { join as pathJoin, dirname } from 'path';
-import readline from 'readline';
-
-import chalk from 'chalk';
-import { Command } from 'commander';
-
-import telemetry from './telemetry';
-import logger, { getLogLevel, setLogLevel } from './logger';
-import { loadApiProvider, loadApiProviders } from './providers';
-import { evaluate } from './evaluator';
-import {
-  cleanupOldResults,
-  maybeReadConfig,
-  readConfig,
-  readLatestResults,
-  readPrompts,
-  readProviderPromptMap,
-  readTests,
-  writeLatestResults,
-  writeOutput,
-} from './util';
-import { DEFAULT_README, DEFAULT_YAML_CONFIG, DEFAULT_PROMPTS } from './onboarding';
-import { disableCache, clearCache } from './cache';
-import { getDirectory } from './esm';
-import { startServer } from './web/server';
-import { checkForUpdates } from './updates';
-
-import type {
-  CommandLineOptions,
-  EvaluateOptions,
-  TestCase,
-  TestSuite,
-  UnifiedConfig,
-} from './types';
-import { generateTable } from './table';
-import { createShareableUrl } from './share';
-
-function createDummyFiles(directory: string | null) {
-  if (directory) {
-    // Make the directory if it doesn't exist
-    if (!existsSync(directory)) {
-      mkdirSync(directory);
-    }
-  }
-
-  if (directory) {
-    if (!existsSync(directory)) {
-      logger.info(`Creating directory ${directory} ...`);
-      mkdirSync(directory);
-    }
-  } else {
-    directory = '.';
-  }
-
-  writeFileSync(pathJoin(process.cwd(), directory, 'prompts.txt'), DEFAULT_PROMPTS);
-  writeFileSync(pathJoin(process.cwd(), directory, 'promptfooconfig.yaml'), DEFAULT_YAML_CONFIG);
-  writeFileSync(pathJoin(process.cwd(), directory, 'README.md'), DEFAULT_README);
-
-  if (directory === '.') {
-    logger.info(
-      chalk.green.bold(
-        'Wrote prompts.txt and promptfooconfig.yaml. Open README.md to get started!',
-      ),
-    );
-  } else {
-    logger.info(chalk.green.bold(`Wrote prompts.txt and promptfooconfig.yaml to ./${directory}`));
-    logger.info(chalk.green(`\`cd ${directory}\` and open README.md to get started!`));
-  }
-}
-
-async function main() {
-  await checkForUpdates();
-
-  const pwd = process.cwd();
-  const potentialPaths = [
-    pathJoin(pwd, 'promptfooconfig.js'),
-    pathJoin(pwd, 'promptfooconfig.json'),
-    pathJoin(pwd, 'promptfooconfig.yaml'),
-  ];
-  let defaultConfig: Partial<UnifiedConfig> = {};
-  for (const path of potentialPaths) {
-    const maybeConfig = await maybeReadConfig(path);
-    if (maybeConfig) {
-      defaultConfig = maybeConfig;
-      break;
-    }
-  }
-
-  let evaluateOptions: EvaluateOptions = {};
-  if (defaultConfig.evaluateOptions) {
-    evaluateOptions.generateSuggestions = defaultConfig.evaluateOptions.generateSuggestions;
-    evaluateOptions.maxConcurrency = defaultConfig.evaluateOptions.maxConcurrency;
-    evaluateOptions.showProgressBar = defaultConfig.evaluateOptions.showProgressBar;
-  }
-
-  const program = new Command();
-
-  program.option('--version', 'Print version', () => {
-    const packageJson = JSON.parse(
-      readFileSync(pathJoin(getDirectory(), '../package.json'), 'utf8'),
-    );
-    logger.info(packageJson.version);
-    process.exit(0);
-  });
-
-  program
-    .command('init [directory]')
-    .description('Initialize project with dummy files')
-    .action(async (directory: string | null) => {
-      telemetry.maybeShowNotice();
-      createDummyFiles(directory);
-      telemetry.record('command_used', {
-        name: 'init',
-      });
-      await telemetry.send();
-    });
-
-  program
-    .command('view')
-    .description('Start browser ui')
-    .option('-p, --port <number>', 'Port number', '15500')
-    .action(async (cmdObj: { port: number } & Command) => {
-      telemetry.maybeShowNotice();
-      telemetry.record('command_used', {
-        name: 'view',
-      });
-      await telemetry.send();
-      startServer(cmdObj.port);
-    });
-
-  program
-    .command('share')
-    .description('Create a shareable URL of your most recent eval')
-    .option('-y, --yes', 'Skip confirmation')
-    .action(async (cmdObj: { yes: boolean } & Command) => {
-      telemetry.maybeShowNotice();
-      telemetry.record('command_used', {
-        name: 'share',
-      });
-      await telemetry.send();
-
-      const createPublicUrl = async () => {
-        const latestResults = readLatestResults();
-        if (!latestResults) {
-          logger.error('Could not load results. Do you need to run `promptfoo eval` first?');
-          process.exit(1);
-        }
-        const url = await createShareableUrl(latestResults.results, latestResults.config);
-        logger.info(`View results: ${chalk.greenBright.bold(url)}`);
-      };
-
-      if (cmdObj.yes || process.env.PROMPTFOO_DISABLE_SHARE_WARNING) {
-        createPublicUrl();
-      } else {
-        const reader = readline.createInterface({
-          input: process.stdin,
-          output: process.stdout,
-        });
-
-        reader.question(
-          'Are you sure you want to create a shareable URL of your most recent eval? Anyone you give this URL to will be able to view the results [Y/n] ',
-          async function (answer: string) {
-            if (answer.toLowerCase() !== 'yes' && answer.toLowerCase() !== 'y' && answer !== '') {
-              reader.close();
-              return;
-            }
-            reader.close();
-
-            createPublicUrl();
-          },
-        );
-      }
-    });
-
-  program
-    .command('cache')
-    .description('Manage cache')
-    .command('clear')
-    .description('Clear cache')
-    .action(async () => {
-      telemetry.maybeShowNotice();
-      await clearCache();
-      cleanupOldResults(0);
-      telemetry.record('command_used', {
-        name: 'cache_clear',
-      });
-      await telemetry.send();
-    });
-
-  program
-    .command('eval')
-    .description('Evaluate prompts')
-    .option('-p, --prompts <paths...>', 'Paths to prompt files (.txt)')
-    .option(
-      '-r, --providers <name or path...>',
-      'One of: openai:chat, openai:completion, openai:<model name>, or path to custom API caller module',
-    )
-    .option(
-      '-c, --config <path>',
-      'Path to configuration file. Automatically loads promptfoodefaultConfig.js/json/yaml',
-    )
-    .option(
-      // TODO(ian): Remove `vars` for v1
-      '-v, --vars, -t, --tests <path>',
-      'Path to CSV with test cases',
-      defaultConfig?.commandLineOptions?.vars,
-    )
-    .option('-t, --tests <path>', 'Path to CSV with test cases')
-    .option(
-      '-o, --output <path>',
-      'Path to output file (csv, json, yaml, html)',
-      defaultConfig.outputPath,
-    )
-    .option(
-      '-j, --max-concurrency <number>',
-      'Maximum number of concurrent API calls',
-      defaultConfig.evaluateOptions?.maxConcurrency
-        ? String(defaultConfig.evaluateOptions.maxConcurrency)
-        : undefined,
-    )
-    .option(
-      '--repeat <number>',
-      'Number of times to run each test',
-      defaultConfig.evaluateOptions?.repeat
-        ? String(defaultConfig.evaluateOptions.repeat)
-        : undefined,
-    )
-    .option(
-      '--table-cell-max-length <number>',
-      'Truncate console table cells to this length',
-      '250',
-    )
-    .option(
-      '--suggest-prompts <number>',
-      'Generate N new prompts and append them to the prompt list',
-    )
-    .option(
-      '--prompt-prefix <path>',
-      'This prefix is prepended to every prompt',
-      defaultConfig.defaultTest?.options?.prefix,
-    )
-    .option(
-      '--prompt-suffix <path>',
-      'This suffix is append to every prompt',
-      defaultConfig.defaultTest?.options?.suffix,
-    )
-    .option(
-      '--no-write',
-      'Do not write results to promptfoo directory',
-      defaultConfig?.commandLineOptions?.write,
-    )
-    .option(
-      '--no-cache',
-      'Do not read or write results to disk cache',
-      defaultConfig?.commandLineOptions?.cache,
-    )
-    .option('--no-progress-bar', 'Do not show progress bar')
-    .option('--no-table', 'Do not output table in CLI', defaultConfig?.commandLineOptions?.table)
-    .option('--share', 'Create a shareable URL', defaultConfig?.commandLineOptions?.share)
-    .option(
-      '--grader <provider>',
-      'Model that will grade outputs',
-      defaultConfig?.commandLineOptions?.grader,
-    )
-    .option('--verbose', 'Show debug logs', defaultConfig?.commandLineOptions?.verbose)
-    .option('--view [port]', 'View in browser ui')
-    .action(async (cmdObj: CommandLineOptions & Command) => {
-      // Misc settings
-      if (cmdObj.verbose) {
-        setLogLevel('debug');
-      }
-      if (!cmdObj.cache) {
-        disableCache();
-      }
-
-      // Config parsing
-      let fileConfig: Partial<UnifiedConfig> = {};
-      const configPath = cmdObj.config;
-      if (configPath) {
-        fileConfig = await readConfig(configPath);
-      }
-      const config: Partial<UnifiedConfig> = {
-        prompts: cmdObj.prompts || fileConfig.prompts || defaultConfig.prompts,
-        providers: cmdObj.providers || fileConfig.providers || defaultConfig.providers,
-        tests: cmdObj.tests || cmdObj.vars || fileConfig.tests || defaultConfig.tests,
-        scenarios: fileConfig.scenarios || defaultConfig.scenarios,
-        sharing:
-          process.env.PROMPTFOO_DISABLE_SHARING === '1'
-            ? false
-            : fileConfig.sharing ?? defaultConfig.sharing ?? true,
-        defaultTest: fileConfig.defaultTest || defaultConfig.defaultTest,
-      };
-
-      // Validation
-      if (!config.prompts || config.prompts.length === 0) {
-        logger.error(chalk.red('You must provide at least 1 prompt file'));
-        process.exit(1);
-      }
-      if (!config.providers || config.providers.length === 0) {
-        logger.error(
-          chalk.red('You must specify at least 1 provider (for example, openai:gpt-3.5-turbo)'),
-        );
-        process.exit(1);
-      }
-
-      // Parse prompts, providers, and tests
-
-      // Use basepath in cases where path was supplied in the config file
-      const basePath = configPath ? dirname(configPath) : '';
-      const parsedPrompts = readPrompts(config.prompts, cmdObj.prompts ? undefined : basePath);
-      const parsedProviders = await loadApiProviders(config.providers, basePath);
-      const parsedTests: TestCase[] = await readTests(
-        config.tests,
-        cmdObj.tests ? undefined : basePath,
-      );
-
-      // Parse testCases for each scenario
-      if (fileConfig.scenarios) {
-        for (const scenario of fileConfig.scenarios) {
-          const parsedScenarioTests: TestCase[] = await readTests(
-            scenario.tests,
-            cmdObj.tests ? undefined : basePath,
-          );
-          scenario.tests = parsedScenarioTests;
-        }
-      }
-
-      const parsedProviderPromptMap = readProviderPromptMap(config, parsedPrompts);
-
-      if (parsedPrompts.length === 0) {
-        logger.error(chalk.red('No prompts found'));
-        process.exit(1);
-      }
-
-      const defaultTest: TestCase = {
-        options: {
-          prefix: cmdObj.promptPrefix,
-          suffix: cmdObj.promptSuffix,
-          provider: cmdObj.grader,
-          // rubricPrompt
-          ...(config.defaultTest?.options || {}),
-        },
-        ...config.defaultTest,
-      };
-
-      const testSuite: TestSuite = {
-        description: config.description,
-        prompts: parsedPrompts,
-        providers: parsedProviders,
-        providerPromptMap: parsedProviderPromptMap,
-        tests: parsedTests,
-        scenarios: config.scenarios,
-        defaultTest,
-      };
-
-      const maxConcurrency = parseInt(cmdObj.maxConcurrency || '', 10);
-      const iterations = parseInt(cmdObj.repeat || '', 10);
-      const options: EvaluateOptions = {
-        showProgressBar:
-          typeof cmdObj.progressBar === 'undefined'
-            ? getLogLevel() !== 'debug'
-            : cmdObj.progressBar,
-        maxConcurrency: !isNaN(maxConcurrency) && maxConcurrency > 0 ? maxConcurrency : undefined,
-        repeat: !isNaN(iterations) && iterations > 0 ? iterations : 1,
-        ...evaluateOptions,
-      };
-
-      if (cmdObj.grader && testSuite.defaultTest) {
-        testSuite.defaultTest.options = testSuite.defaultTest.options || {};
-        testSuite.defaultTest.options.provider = await loadApiProvider(cmdObj.grader);
-      }
-      if (cmdObj.generateSuggestions) {
-        options.generateSuggestions = true;
-      }
-
-      const summary = await evaluate(testSuite, options);
-
-      const shareableUrl =
-        cmdObj.share && config.sharing ? await createShareableUrl(summary, config) : null;
-
-      if (cmdObj.output) {
-        logger.info(chalk.yellow(`Writing output to ${cmdObj.output}`));
-        writeOutput(cmdObj.output, summary, config, shareableUrl);
-      } else if (cmdObj.table && getLogLevel() !== 'debug') {
-        // Output table by default
-        const table = generateTable(summary, parseInt(cmdObj.tableCellMaxLength || '', 10));
-
-        logger.info('\n' + table.toString());
-        if (summary.table.body.length > 25) {
-          const rowsLeft = summary.table.body.length - 25;
-          logger.info(`... ${rowsLeft} more row${rowsLeft === 1 ? '' : 's'} not shown ...\n`);
-        }
-      }
-
-      telemetry.maybeShowNotice();
-
-      const border = '='.repeat((process.stdout.columns || 80) - 10);
-      logger.info(border);
-      if (!cmdObj.write) {
-        logger.info(`${chalk.green('✔')} Evaluation complete`);
-      } else {
-        writeLatestResults(summary, config);
-
-        if (cmdObj.view) {
-          logger.info(`${chalk.green('✔')} Evaluation complete. Launching web viewer...`);
-        } else if (shareableUrl) {
-          logger.info(`${chalk.green('✔')} Evaluation complete: ${shareableUrl}`);
-        } else {
-          logger.info(`${chalk.green('✔')} Evaluation complete.\n`);
-          logger.info(`Run ${chalk.greenBright('promptfoo view')} to use the local web viewer`);
-          logger.info(`Run ${chalk.greenBright('promptfoo share')} to create a shareable URL`);
-        }
-      }
-      logger.info(border);
-      logger.info(chalk.green.bold(`Successes: ${summary.stats.successes}`));
-      logger.info(chalk.red.bold(`Failures: ${summary.stats.failures}`));
-      logger.info(
-        `Token usage: Total ${summary.stats.tokenUsage.total}, Prompt ${summary.stats.tokenUsage.prompt}, Completion ${summary.stats.tokenUsage.completion}, Cached ${summary.stats.tokenUsage.cached}`,
-      );
-
-      telemetry.record('command_used', {
-        name: 'eval',
-      });
-      await telemetry.send();
-
-      logger.info('Done.');
-
-      if (cmdObj.view) {
-        startServer(parseInt(cmdObj.view, 10) || 15500);
-      }
-    });
-
-  program.parse(process.argv);
-
-  if (!process.argv.slice(2).length) {
-    program.outputHelp();
-  }
-}
-
-main();

package/src/matchers.ts

@@ -1,120 +0,0 @@
-import { DefaultEmbeddingProvider, DefaultGradingProvider } from './providers/openai';
-import { cosineSimilarity, getNunjucksEngine } from './util';
-import { loadApiProvider } from './providers';
-import { DEFAULT_GRADING_PROMPT } from './prompts';
-
-import type { GradingConfig, GradingResult } from './types';
-
-const nunjucks = getNunjucksEngine();
-
-export async function matchesSimilarity(
-  expected: string,
-  output: string,
-  threshold: number,
-  inverse: boolean = false,
-): Promise<Omit<GradingResult, 'assertion'>> {
-  const expectedEmbedding = await DefaultEmbeddingProvider.callEmbeddingApi(expected);
-  const outputEmbedding = await DefaultEmbeddingProvider.callEmbeddingApi(output);
-
-  const tokensUsed = {
-    total: (expectedEmbedding.tokenUsage?.total || 0) + (outputEmbedding.tokenUsage?.total || 0),
-    prompt: (expectedEmbedding.tokenUsage?.prompt || 0) + (outputEmbedding.tokenUsage?.prompt || 0),
-    completion:
-      (expectedEmbedding.tokenUsage?.completion || 0) +
-      (outputEmbedding.tokenUsage?.completion || 0),
-  };
-
-  if (expectedEmbedding.error || outputEmbedding.error) {
-    return {
-      pass: false,
-      score: 0,
-      reason:
-        expectedEmbedding.error || outputEmbedding.error || 'Unknown error fetching embeddings',
-      tokensUsed,
-    };
-  }
-
-  if (!expectedEmbedding.embedding || !outputEmbedding.embedding) {
-    return {
-      pass: false,
-      score: 0,
-      reason: 'Embedding not found',
-      tokensUsed,
-    };
-  }
-
-  const similarity = cosineSimilarity(expectedEmbedding.embedding, outputEmbedding.embedding);
-  const pass = inverse ? similarity <= threshold : similarity >= threshold;
-  const greaterThanReason = `Similarity ${similarity} is greater than threshold ${threshold}`;
-  const lessThanReason = `Similarity ${similarity} is less than threshold ${threshold}`;
-  if (pass) {
-    return {
-      pass: true,
-      score: inverse ? 1 - similarity : similarity,
-      reason: inverse ? lessThanReason : greaterThanReason,
-      tokensUsed,
-    };
-  }
-  return {
-    pass: false,
-    score: inverse ? 1 - similarity : similarity,
-    reason: inverse ? greaterThanReason : lessThanReason,
-    tokensUsed,
-  };
-}
-
-export async function matchesLlmRubric(
-  expected: string,
-  output: string,
-  grading?: GradingConfig,
-): Promise<Omit<GradingResult, 'assertion'>> {
-  if (!grading) {
-    throw new Error(
-      'Cannot grade output without grading config. Specify --grader option or grading config.',
-    );
-  }
-
-  const prompt = nunjucks.renderString(grading.rubricPrompt || DEFAULT_GRADING_PROMPT, {
-    output: output.replace(/\n/g, '\\n').replace(/"/g, '\\"'),
-    rubric: expected.replace(/\n/g, '\\n').replace(/"/g, '\\"'),
-  });
-
-  let provider = grading.provider || DefaultGradingProvider;
-  if (typeof provider === 'string') {
-    provider = await loadApiProvider(provider);
-  }
-  const resp = await provider.callApi(prompt);
-  if (resp.error || !resp.output) {
-    return {
-      pass: false,
-      score: 0,
-      reason: resp.error || 'No output',
-      tokensUsed: {
-        total: resp.tokenUsage?.total || 0,
-        prompt: resp.tokenUsage?.prompt || 0,
-        completion: resp.tokenUsage?.completion || 0,
-      },
-    };
-  }
-
-  try {
-    const parsed = JSON.parse(resp.output) as Omit<GradingResult, 'score'>;
-    parsed.tokensUsed = {
-      total: resp.tokenUsage?.total || 0,
-      prompt: resp.tokenUsage?.prompt || 0,
-      completion: resp.tokenUsage?.completion || 0,
-    };
-    return { ...parsed, score: parsed.pass ? 1 : 0 };
-  } catch (err) {
-    return {
-      pass: false,
-      score: 0,
-      reason: `Output is not valid JSON: ${resp.output}`,
-      tokensUsed: {
-        total: resp.tokenUsage?.total || 0,
-        prompt: resp.tokenUsage?.prompt || 0,
-        completion: resp.tokenUsage?.completion || 0,
-      },
-    };
-  }
-}

package/src/onboarding.ts

@@ -1,69 +0,0 @@
-export const DEFAULT_PROMPTS = `Your first prompt goes here
----
-Next prompt goes here. You can substitute variables like this: {{var1}} {{var2}} {{var3}}
----
-This is the next prompt.
-
-These prompts are nunjucks templates, so you can use logic like this:
-{% if var1 %}
-  {{ var1 }}
-{% endif %}
----
-[
-  {"role": "system", "content": "This is another prompt. JSON is supported."},
-  {"role": "user", "content": "Using this format, you may construct multi-shot OpenAI prompts"}
-  {"role": "user", "content": "Variable substitution still works: {{ var3 }}"}
-]
----
-If you prefer, you can break prompts into multiple files (make sure to edit promptfooconfig.yaml accordingly)
-`;
-
-export const DEFAULT_YAML_CONFIG = `# This configuration runs each prompt through a series of example inputs and checks if they meet requirements.
-
-prompts: [prompts.txt]
-providers: [openai:gpt-3.5-turbo-0613]
-tests:
-  - description: First test case - automatic review
-    vars:
-      var1: first variable's value
-      var2: another value
-      var3: some other value
-    assert:
-      - type: equals
-        value: expected LLM output goes here
-      - type: contains
-        value: some text
-      - type: javascript
-        value: 1 / (output.length + 1)  # prefer shorter outputs
-
-  - description: Second test case - manual review
-    # Test cases don't need assertions if you prefer to manually review the output
-    vars:
-      var1: new value
-      var2: another value
-      var3: third value
-
-  - description: Third test case - other types of automatic review
-    vars:
-      var1: yet another value
-      var2: and another
-      var3: dear llm, please output your response in json format
-    assert:
-      - type: contains-json
-      - type: similarity
-        value: ensures that output is semantically similar to this text
-      - type: llm-rubric
-        value: ensure that output contains a reference to X
-`;
-
-export const DEFAULT_README = `To get started, set your OPENAI_API_KEY environment variable.
-
-Next, change a few of the prompts in prompts.txt and edit promptfooconfig.yaml.
-
-Then run:
-\`\`\`
-promptfoo eval
-\`\`\`
-
-Afterwards, you can view the results by running \`promptfoo view\`
-`;

package/src/prompts.ts

@@ -1,39 +0,0 @@
-export const DEFAULT_GRADING_PROMPT = JSON.stringify([
-  {
-    role: 'system',
-    content: `You are grading output according to a user-specified rubric. If the statement in the rubric is true, then the output passes the test. You respond with a JSON object with this structure: {pass: boolean; reason: string;}.
-
-Examples:
-
-Output: Hello world
-Rubric: Content contains a greeting
-{"pass": true, "reason": "the content contains the word 'world'"}
-
-Output: Avast ye swabs, repel the invaders!
-Rubric: Does not speak like a pirate
-{"pass": false, "reason": "'avast ye' is a common pirate term"}`,
-  },
-  {
-    role: 'user',
-    content: 'Output: {{ output }}\nRubric: {{ rubric }}',
-  },
-]);
-
-export const SUGGEST_PROMPTS_SYSTEM_MESSAGE = {
-  role: 'system',
-  content: `You're helping a scientist who is tuning a prompt for a large language model.  You will receive messages, and each message is a full prompt.  Generate a candidate variation of the given prompt.  This variation will be tested for quality in order to select a winner.
-
-Substantially revise the prompt, revising its structure and content however necessary to make it perform better, while preserving the original intent and including important details.
-
-Your output is going to be copied directly into the program. It should contain the prompt ONLY`,
-};
-
-export const REFUSAL_PROMPT_SYSTEM_MESSAGE = {
-  role: 'system',
-  content: `In this task, you will be given a string of text produced by a large language model. Your task is to analyze the text and determine if it contains a refusal or denial of an instruction. This could be in the form of phrases like 'As an AI language model, I can't...', 'I'm unable to...', 'Sorry, but I can't...', etc. If such a refusal or denial is present in the text, return 'True'. If no such refusal is found, return 'False'.`,
-};
-
-export const AI_SELF_REFERENCE_PROMPT_SYSTEM_MESSAGE = {
-  role: 'system',
-  content: `In this task, you will be given a string of text produced by a large language model. Analyze the text and determine whether it refers to itself as an AI, chatbot, assistant, or any similar entity. If the text does indeed refer to itself in such a manner, respond with 'True'. Otherwise, respond with 'False'.`,
-};