promptfoo 0.5.1 → 0.7.0

package/src/evaluator.ts

@@ -1,8 +1,11 @@
+import readline from 'node:readline';
+
 import async from 'async';
+import chalk from 'chalk';
 import nunjucks from 'nunjucks';
 
-import { DEFAULT_GRADING_PROMPT } from './prompts.js';
-import { DefaultEmbeddingProvider } from './providers/openai.js';
+import logger from './logger.js';
+import { matchesExpectedValue } from './assertions.js';
 
 import type { SingleBar } from 'cli-progress';
 import type {
@@ -13,9 +16,8 @@ import type {
   EvaluateSummary,
   EvaluateTable,
   Prompt,
-  TokenUsage,
 } from './types.js';
-import { cosineSimilarity } from './util.js';
+import { generatePrompts } from './suggestions.js';
 
 interface RunEvalOptions {
   provider: ApiProvider;
@@ -27,16 +29,8 @@ interface RunEvalOptions {
   colIndex: number;
 }
 
-interface GradingResult {
-  pass: boolean;
-  reason: string;
-  tokensUsed: TokenUsage;
-}
-
 const DEFAULT_MAX_CONCURRENCY = 4;
 
-const SIMILAR_REGEX = /similar(?::|\((\d+(\.\d+)?)\):)/;
-
 class Evaluator {
   options: EvaluateOptions;
   stats: EvaluateStats;
@@ -50,132 +44,11 @@ class Evaluator {
         total: 0,
         prompt: 0,
         completion: 0,
+        cached: 0,
       },
     };
   }
 
-  async gradeOutput(expected: string, output: string): Promise<GradingResult> {
-    const { grading } = this.options;
-
-    if (!grading) {
-      throw new Error(
-        'Cannot grade output without grading config. Specify --grader option or grading config.',
-      );
-    }
-
-    const prompt = nunjucks.renderString(grading.prompt || DEFAULT_GRADING_PROMPT, {
-      content: output,
-      rubric: expected,
-    });
-
-    const resp = await grading.provider.callApi(prompt);
-    if (resp.error || !resp.output) {
-      return {
-        pass: false,
-        reason: resp.error || 'No output',
-        tokensUsed: {
-          total: resp.tokenUsage?.total || 0,
-          prompt: resp.tokenUsage?.prompt || 0,
-          completion: resp.tokenUsage?.completion || 0,
-        },
-      };
-    }
-
-    try {
-      const parsed = JSON.parse(resp.output) as GradingResult;
-      parsed.tokensUsed = {
-        total: resp.tokenUsage?.total || 0,
-        prompt: resp.tokenUsage?.prompt || 0,
-        completion: resp.tokenUsage?.completion || 0,
-      };
-      return parsed;
-    } catch (err) {
-      return {
-        pass: false,
-        reason: `Output is not valid JSON: ${resp.output}`,
-        tokensUsed: {
-          total: resp.tokenUsage?.total || 0,
-          prompt: resp.tokenUsage?.prompt || 0,
-          completion: resp.tokenUsage?.completion || 0,
-        },
-      };
-    }
-  }
-
-  async checkSimilarity(
-    expected: string,
-    output: string,
-    threshold: number,
-  ): Promise<GradingResult> {
-    const expectedEmbedding = await DefaultEmbeddingProvider.callEmbeddingApi(expected);
-    const outputEmbedding = await DefaultEmbeddingProvider.callEmbeddingApi(output);
-
-    const tokensUsed = {
-      total: (expectedEmbedding.tokenUsage?.total || 0) + (outputEmbedding.tokenUsage?.total || 0),
-      prompt:
-        (expectedEmbedding.tokenUsage?.prompt || 0) + (outputEmbedding.tokenUsage?.prompt || 0),
-      completion:
-        (expectedEmbedding.tokenUsage?.completion || 0) +
-        (outputEmbedding.tokenUsage?.completion || 0),
-    };
-
-    if (expectedEmbedding.error || outputEmbedding.error) {
-      return {
-        pass: false,
-        reason:
-          expectedEmbedding.error || outputEmbedding.error || 'Unknown error fetching embeddings',
-        tokensUsed,
-      };
-    }
-
-    if (!expectedEmbedding.embedding || !outputEmbedding.embedding) {
-      return {
-        pass: false,
-        reason: 'Embedding not found',
-        tokensUsed,
-      };
-    }
-
-    const similarity = cosineSimilarity(expectedEmbedding.embedding, outputEmbedding.embedding);
-    if (similarity < threshold) {
-      return {
-        pass: false,
-        reason: `Similarity ${similarity} is less than threshold ${threshold}`,
-        tokensUsed,
-      };
-    }
-    return {
-      pass: true,
-      reason: `Similarity ${similarity} is greater than threshold ${threshold}`,
-      tokensUsed,
-    };
-  }
-
-  async checkExpectedValue(
-    expected: string,
-    output: string,
-  ): Promise<{ pass: boolean; reason?: string }> {
-    const match = expected.match(SIMILAR_REGEX);
-
-    if (match) {
-      const threshold = parseFloat(match[1]) || 0.8;
-      const rest = expected.replace(SIMILAR_REGEX, '').trim();
-      return this.checkSimilarity(rest, output, threshold);
-    } else if (expected.startsWith('eval:')) {
-      const evalBody = expected.slice(5);
-      const evalFunction = new Function('output', `return ${evalBody}`);
-      return { pass: evalFunction(output) };
-    } else if (expected.startsWith('grade:')) {
-      return this.gradeOutput(expected.slice(6), output);
-    } else {
-      const pass = expected === output;
-      return {
-        pass,
-        reason: pass ? undefined : `Expected: ${expected}, Output: ${output}`,
-      };
-    }
-  }
-
   async runEval({
     provider,
     prompt,
@@ -207,7 +80,7 @@ class Evaluator {
         ret.error = response.error;
       } else if (response.output) {
         const checkResult = vars.__expected
-          ? await this.checkExpectedValue(vars.__expected, response.output)
+          ? await matchesExpectedValue(vars.__expected, response.output, this.options)
           : { pass: true };
         if (!checkResult.pass) {
           ret.error = checkResult.reason || `Expected: ${vars.__expected}`;
@@ -222,6 +95,7 @@ class Evaluator {
       this.stats.tokenUsage.total += response.tokenUsage?.total || 0;
       this.stats.tokenUsage.prompt += response.tokenUsage?.prompt || 0;
       this.stats.tokenUsage.completion += response.tokenUsage?.completion || 0;
+      this.stats.tokenUsage.cached += response.tokenUsage?.cached || 0;
 
       if (ret.success) {
         this.stats.successes++;
@@ -243,6 +117,48 @@ class Evaluator {
     const options = this.options;
     const prompts: Prompt[] = [];
 
+    if (options.prompt?.generateSuggestions) {
+      logger.info(`Generating prompt variations...`);
+      const { prompts: newPrompts, error } = await generatePrompts(options.prompts[0], 1);
+      if (error || !newPrompts) {
+        throw new Error(`Failed to generate prompts: ${error}`);
+      }
+
+      logger.info(chalk.blue('Generated prompts:'));
+      let numAdded = 0;
+      for (const prompt of newPrompts) {
+        logger.info('--------------------------------------------------------');
+        logger.info(`${prompt}`);
+        logger.info('--------------------------------------------------------');
+
+        // Ask the user if they want to continue
+        await new Promise((resolve) => {
+          const rl = readline.createInterface({
+            input: process.stdin,
+            output: process.stdout,
+          });
+          rl.question(
+            `${chalk.blue('Do you want to test this prompt?')} (y/N): `,
+            async (answer) => {
+              rl.close();
+              if (answer.toLowerCase().startsWith('y')) {
+                options.prompts.push(prompt);
+                numAdded++;
+              } else {
+                logger.info('Skipping this prompt.');
+              }
+              resolve(true);
+            },
+          );
+        });
+      }
+
+      if (numAdded < 1) {
+        logger.info(chalk.red('No prompts selected. Aborting.'));
+        process.exit(1);
+      }
+    }
+
     for (const promptContent of options.prompts) {
       for (const provider of options.providers) {
         const display =
@@ -255,16 +171,20 @@ class Evaluator {
     }
 
     const vars = options.vars && options.vars.length > 0 ? options.vars : [{}];
-    const varsWithExpectedKeyRemoved = vars.map((v) => {
+    const varsWithSpecialColsRemoved = vars.map((v) => {
       const ret = { ...v };
-      delete ret.__expected;
+      Object.keys(ret).forEach((key) => {
+        if (key.startsWith('__')) {
+          delete ret[key];
+        }
+      });
       return ret;
     });
     const isTest = vars[0].__expected;
     const table: EvaluateTable = {
       head: {
         prompts: prompts.map((p) => p.display),
-        vars: Object.keys(varsWithExpectedKeyRemoved[0]),
+        vars: Object.keys(varsWithSpecialColsRemoved[0]),
       },
       body: [],
     };
@@ -292,11 +212,15 @@ class Evaluator {
     let rowIndex = 0;
     for (const row of vars) {
       let colIndex = 0;
+
+      const prependToPrompt = row.__prefix || options.prompt?.prefix || '';
+      const appendToPrompt = row.__suffix || options.prompt?.suffix || '';
+
       for (const promptContent of options.prompts) {
         for (const provider of options.providers) {
           runEvalOptions.push({
             provider,
-            prompt: promptContent,
+            prompt: prependToPrompt + promptContent + appendToPrompt,
             vars: row,
             includeProviderId: options.providers.length > 1,
             rowIndex,

package/src/index.ts

@@ -1,8 +1,12 @@
 import { evaluate as doEvaluate } from './evaluator.js';
 import { loadApiProvider } from './providers.js';
+import assertions from './assertions.js';
+import providers from './providers.js';
 
 import type { ApiProvider, EvaluateOptions, EvaluateSummary } from './types.js';
 
+export * from './types.js';
+
 async function evaluate(
   providers: (string | ApiProvider)[] | (string | ApiProvider),
   options: Omit<EvaluateOptions, 'providers'>,
@@ -30,6 +34,14 @@ async function evaluate(
   });
 }
 
+module.exports = {
+  evaluate,
+  assertions,
+  providers,
+};
+
 export default {
   evaluate,
+  assertions,
+  providers,
 };

package/src/main.ts

@@ -14,6 +14,7 @@ import { getDirectory } from './esm.js';
 import { init } from './web/server.js';
 
 import type { CommandLineOptions, EvaluateOptions, VarMapping } from './types.js';
+import { disableCache } from './cache.js';
 
 function createDummyFiles(directory: string | null) {
   if (directory) {
@@ -34,7 +35,7 @@ These prompts are nunjucks templates, so you can use logic like this:
 {% endif %}`;
   const dummyVars =
     'var1,var2,var3\nvalue1,value2,value3\nanother value1,another value2,another value3';
-  const dummyConfig = `export default {
+  const dummyConfig = `module.exports = {
   prompts: ['prompts.txt'],
   providers: ['openai:gpt-3.5-turbo'],
   vars: 'vars.csv',
@@ -79,6 +80,10 @@ async function main() {
     defaultConfig = (await import(pathJoin(process.cwd(), './promptfooconfig.js'))).default;
     logger.info('Loaded default config from promptfooconfig.js');
   }
+  if (existsSync('promptfooconfig.json')) {
+    defaultConfig = JSON.parse(readFileSync('promptfooconfig.json', 'utf-8'));
+    logger.info('Loaded default config from promptfooconfig.json');
+  }
 
   const program = new Command();
 
@@ -143,15 +148,26 @@ async function main() {
       'Truncate console table cells to this length',
       '250',
     )
+    .option(
+      '--suggest-prompts <number>',
+      'Generate N new prompts and append them to the prompt list',
+    )
+    .option(
+      '--prompt-prefix <path>',
+      'This prefix is prepended to every prompt',
+      defaultConfig.promptPrefix,
+    )
+    .option(
+      '--prompt-suffix <path>',
+      'This suffix is append to every prompt',
+      defaultConfig.promptSuffix,
+    )
     .option('--no-write', 'Do not write results to promptfoo directory')
+    .option('--no-cache', 'Do not read or write results to disk cache')
     .option('--grader', 'Model that will grade outputs', defaultConfig.grader)
     .option('--verbose', 'Show debug logs', defaultConfig.verbose)
-    .option('--view', 'View in browser ui')
+    .option('--view [port]', 'View in browser ui')
     .action(async (cmdObj: CommandLineOptions & Command) => {
-      if (cmdObj.verbose) {
-        setLogLevel('debug');
-      }
-
       const configPath = cmdObj.config;
       let config = {};
       if (configPath) {
@@ -169,6 +185,13 @@ async function main() {
         }
       }
 
+      if (cmdObj.verbose) {
+        setLogLevel('debug');
+      }
+      if (!cmdObj.cache) {
+        disableCache();
+      }
+
       let vars: VarMapping[] = [];
       if (cmdObj.vars) {
         vars = readVars(cmdObj.vars);
@@ -184,6 +207,10 @@ async function main() {
         providers,
         showProgressBar: true,
         maxConcurrency: !isNaN(maxConcurrency) && maxConcurrency > 0 ? maxConcurrency : undefined,
+        prompt: {
+          prefix: cmdObj.promptPrefix,
+          suffix: cmdObj.promptSuffix,
+        },
         ...config,
       };
 
@@ -192,6 +219,9 @@ async function main() {
           provider: await loadApiProvider(cmdObj.grader),
         };
       }
+      if (cmdObj.generateSuggestions) {
+        options.prompt!.generateSuggestions = true;
+      }
 
       const summary = await evaluate(options);
 
@@ -238,7 +268,7 @@ async function main() {
 
         logger.info('\n' + table.toString());
       }
-      if (cmdObj.noWrite || cmdObj.view) {
+      if (cmdObj.view || !cmdObj.write) {
         logger.info('Evaluation complete');
       } else {
         writeLatestResults(summary);
@@ -247,12 +277,12 @@ async function main() {
       logger.info(chalk.green.bold(`Successes: ${summary.stats.successes}`));
       logger.info(chalk.red.bold(`Failures: ${summary.stats.failures}`));
       logger.info(
-        `Token usage: Total ${summary.stats.tokenUsage.total} Prompt ${summary.stats.tokenUsage.prompt} Completion ${summary.stats.tokenUsage.completion}`,
+        `Token usage: Total ${summary.stats.tokenUsage.total}, Prompt ${summary.stats.tokenUsage.prompt}, Completion ${summary.stats.tokenUsage.completion}, Cached ${summary.stats.tokenUsage.cached}`,
       );
       logger.info('Done.');
 
       if (cmdObj.view) {
-        init(15500);
+        init(parseInt(cmdObj.view, 10) || 15500);
       }
     });
 

package/src/prompts.ts

@@ -18,3 +18,12 @@ Rubric: Does not speak like a pirate
     content: 'Content: {{ content }}\nRubric: {{ rubric }}',
   },
 ]);
+
+export const SUGGEST_PROMPTS_SYSTEM_MESSAGE = {
+  role: 'system',
+  content: `You're helping a scientist who is tuning a prompt for a large language model.  You will receive messages, and each message is a full prompt.  Generate a candidate variation of the given prompt.  This variation will be tested for quality in order to select a winner.
+
+Substantially revise the prompt, revising its structure and content however necessary to make it perform better, while preserving the original intent and including important details.
+
+Your output is going to be copied directly into the program. It should contain the prompt ONLY`,
+};

package/src/providers/localai.ts

@@ -1,5 +1,5 @@
 import logger from '../logger.js';
-import { fetchWithTimeout } from '../util.js';
+import { fetchJsonWithCache } from '../cache.js';
 import { REQUEST_TIMEOUT_MS } from './shared.js';
 
 import type { ApiProvider, ProviderResponse } from '../types.js';
@@ -36,9 +36,10 @@ export class LocalAiChatProvider extends LocalAiGenericProvider {
     };
     logger.debug(`Calling LocalAI API: ${JSON.stringify(body)}`);
 
-    let response, data;
+    let data,
+      cached = false;
     try {
-      response = await fetchWithTimeout(
+      ({ data, cached } = (await fetchJsonWithCache(
         `${this.apiBaseUrl}/chat/completions`,
         {
           method: 'POST',
@@ -48,9 +49,7 @@ export class LocalAiChatProvider extends LocalAiGenericProvider {
           body: JSON.stringify(body),
         },
         REQUEST_TIMEOUT_MS,
-      );
-
-      data = (await response.json()) as unknown as any;
+      )) as unknown as any);
     } catch (err) {
       return {
         error: `API call error: ${String(err)}`,
@@ -78,9 +77,10 @@ export class LocalAiCompletionProvider extends LocalAiGenericProvider {
     };
     logger.debug(`Calling LocalAI API: ${JSON.stringify(body)}`);
 
-    let response, data;
+    let data,
+      cached = false;
     try {
-      response = await fetchWithTimeout(
+      ({ data, cached } = (await fetchJsonWithCache(
         `${this.apiBaseUrl}/completions`,
         {
           method: 'POST',
@@ -90,9 +90,7 @@ export class LocalAiCompletionProvider extends LocalAiGenericProvider {
           body: JSON.stringify(body),
         },
         REQUEST_TIMEOUT_MS,
-      );
-
-      data = (await response.json()) as unknown as any;
+      )) as unknown as any);
     } catch (err) {
       return {
         error: `API call error: ${String(err)}`,