promptfoo 0.11.0 → 0.13.0

package/src/main.ts

@@ -2,11 +2,11 @@
 import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs';
 import { join as pathJoin } from 'path';
 
-import Table from 'cli-table3';
 import chalk from 'chalk';
 import { Command } from 'commander';
 
-import logger, { setLogLevel } from './logger';
+import telemetry from './telemetry';
+import logger, { getLogLevel, setLogLevel } from './logger';
 import { loadApiProvider, loadApiProviders } from './providers';
 import { evaluate } from './evaluator';
 import {
@@ -17,9 +17,11 @@ import {
   writeLatestResults,
   writeOutput,
 } from './util';
+import { DEFAULT_README, DEFAULT_YAML_CONFIG, DEFAULT_PROMPTS } from './onboarding';
+import { disableCache } from './cache';
 import { getDirectory } from './esm';
 import { init } from './web/server';
-import { disableCache } from './cache';
+import { checkForUpdates } from './updates';
 
 import type {
   CommandLineOptions,
@@ -28,7 +30,7 @@ import type {
   TestSuite,
   UnifiedConfig,
 } from './types';
-import { DEFAULT_README, DEFAULT_YAML_CONFIG, DEFAULT_PROMPTS } from './onboarding';
+import { generateTable } from './table';
 
 function createDummyFiles(directory: string | null) {
   if (directory) {
@@ -60,6 +62,8 @@ function createDummyFiles(directory: string | null) {
 }
 
 async function main() {
+  await checkForUpdates();
+
   const pwd = process.cwd();
   const potentialPaths = [
     pathJoin(pwd, 'promptfooconfig.js'),
@@ -95,15 +99,23 @@ async function main() {
   program
     .command('init [directory]')
     .description('Initialize project with dummy files')
-    .action((directory: string | null) => {
+    .action(async (directory: string | null) => {
       createDummyFiles(directory);
+      telemetry.record('command_used', {
+        name: 'init',
+      });
+      await telemetry.send();
     });
 
   program
     .command('view')
     .description('Start browser ui')
     .option('-p, --port <number>', 'Port number', '15500')
-    .action((cmdObj: { port: number } & Command) => {
+    .action(async (cmdObj: { port: number } & Command) => {
+      telemetry.record('command_used', {
+        name: 'view',
+      });
+      await telemetry.send();
       init(cmdObj.port);
     });
 
@@ -111,10 +123,9 @@ async function main() {
     .command('eval')
     .description('Evaluate prompts')
     .requiredOption('-p, --prompts <paths...>', 'Paths to prompt files (.txt)', config.prompts)
-    .requiredOption(
+    .option(
       '-r, --providers <name or path...>',
       'One of: openai:chat, openai:completion, openai:<model name>, or path to custom API caller module',
-      config?.providers,
     )
     .option(
       '-c, --config <path>',
@@ -231,7 +242,7 @@ async function main() {
       };
 
       const options: EvaluateOptions = {
-        showProgressBar: true,
+        showProgressBar: getLogLevel() !== 'debug',
         maxConcurrency: !isNaN(maxConcurrency) && maxConcurrency > 0 ? maxConcurrency : undefined,
         ...evaluateOptions,
       };
@@ -249,43 +260,9 @@ async function main() {
       if (cmdObj.output) {
         logger.info(chalk.yellow(`Writing output to ${cmdObj.output}`));
         writeOutput(cmdObj.output, summary);
-      } else {
+      } else if (getLogLevel() !== 'debug') {
         // Output table by default
-        const maxWidth = process.stdout.columns ? process.stdout.columns - 10 : 120;
-        const head = summary.table.head;
-        const headLength = head.prompts.length + head.vars.length;
-        const table = new Table({
-          head: [...head.prompts, ...head.vars],
-          colWidths: Array(headLength).fill(Math.floor(maxWidth / headLength)),
-          wordWrap: true,
-          wrapOnWordBoundary: false,
-          style: {
-            head: ['blue', 'bold'],
-          },
-        });
-        // Skip first row (header) and add the rest. Color PASS/FAIL
-        for (const row of summary.table.body.slice(0, 25)) {
-          table.push([
-            ...row.vars,
-            ...row.outputs.map((col) => {
-              const tableCellMaxLength = parseInt(cmdObj.tableCellMaxLength || '', 10);
-              if (!isNaN(tableCellMaxLength) && col.length > tableCellMaxLength) {
-                col = col.slice(0, tableCellMaxLength) + '...';
-              }
-              if (col.startsWith('[PASS]')) {
-                // color '[PASS]' green
-                return chalk.green.bold(col.slice(0, 6)) + col.slice(6);
-              } else if (col.startsWith('[FAIL]')) {
-                // color everything red up until '---'
-                return col
-                  .split('---')
-                  .map((c, idx) => (idx === 0 ? chalk.red.bold(c) : c))
-                  .join('---');
-              }
-              return col;
-            }),
-          ]);
-        }
+        const table = generateTable(summary, parseInt(cmdObj.tableCellMaxLength || '', 10));
 
         logger.info('\n' + table.toString());
         if (summary.table.body.length > 25) {
@@ -293,12 +270,20 @@ async function main() {
           logger.info(`... ${rowsLeft} more row${rowsLeft === 1 ? '' : 's'} not shown ...\n`);
         }
       }
+
+      const border = '='.repeat(process.stdout.columns - 10);
+      logger.info(border);
       if (cmdObj.view || !cmdObj.write) {
-        logger.info('Evaluation complete');
+        logger.info(`${chalk.green('✔')} Evaluation complete`);
       } else {
-        writeLatestResults(summary);
-        logger.info(`Evaluation complete. To use web viewer, run ${chalk.green('promptfoo view')}`);
+        writeLatestResults(summary, config);
+        logger.info(
+          `${chalk.green('✔')} Evaluation complete. To use web viewer, run ${chalk.green(
+            'promptfoo view',
+          )}`,
+        );
       }
+      logger.info(border);
       logger.info(chalk.green.bold(`Successes: ${summary.stats.successes}`));
       logger.info(chalk.red.bold(`Failures: ${summary.stats.failures}`));
       logger.info(
@@ -306,6 +291,11 @@ async function main() {
       );
       logger.info('Done.');
 
+      telemetry.record('command_used', {
+        name: 'eval',
+      });
+      await telemetry.send();
+
       if (cmdObj.view) {
         init(parseInt(cmdObj.view, 10) || 15500);
       }

package/src/onboarding.ts

@@ -9,6 +9,12 @@ These prompts are nunjucks templates, so you can use logic like this:
   {{ var1 }}
 {% endif %}
 ---
+[
+  {"role": "system", "content": "Use JSON too for more complex payloads"},
+  {"role": "user", "content": "Such as multi-shot prompts"}
+  {"role": "user", "content": "Variable substitution still works: {{ var3 }}"}
+]
+---
 If you prefer, you can break prompts into multiple files (make sure to edit promptfooconfig.yaml accordingly)
 `;
 

package/src/providers/openai.ts

@@ -7,7 +7,13 @@ import type { ApiProvider, ProviderEmbeddingResponse, ProviderResponse } from '.
 const DEFAULT_OPENAI_HOST = 'api.openai.com';
 
 interface OpenAiCompletionOptions {
-  temperature: number;
+  temperature?: number;
+  functions?: {
+    name: string;
+    description?: string;
+    parameters: any;
+  }[];
+  function_call?: 'none' | 'auto';
 }
 
 class OpenAiGenericProvider implements ApiProvider {
@@ -112,11 +118,14 @@ export class OpenAiCompletionProvider extends OpenAiGenericProvider {
     'text-ada-001',
   ];
 
-  constructor(modelName: string, apiKey?: string) {
+  options: OpenAiCompletionOptions;
+
+  constructor(modelName: string, apiKey?: string, context?: OpenAiCompletionOptions) {
     if (!OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.includes(modelName)) {
       logger.warn(`Using unknown OpenAI completion model: ${modelName}`);
     }
     super(modelName, apiKey);
+    this.options = context || {};
   }
 
   async callApi(prompt: string, options?: OpenAiCompletionOptions): Promise<ProviderResponse> {
@@ -138,7 +147,10 @@ export class OpenAiCompletionProvider extends OpenAiGenericProvider {
       model: this.modelName,
       prompt,
       max_tokens: parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
-      temperature: options?.temperature ?? parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      temperature:
+        options?.temperature ??
+        this.options.temperature ??
+        parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
       stop,
     };
     logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
@@ -186,17 +198,22 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
   static OPENAI_CHAT_MODELS = [
     'gpt-4',
     'gpt-4-0314',
+    'gpt-4-0613',
     'gpt-4-32k',
     'gpt-4-32k-0314',
     'gpt-3.5-turbo',
     'gpt-3.5-turbo-0301',
+    'gpt-3.5-turbo-0613',
   ];
 
-  constructor(modelName: string, apiKey?: string) {
+  options: OpenAiCompletionOptions;
+
+  constructor(modelName: string, apiKey?: string, context?: OpenAiCompletionOptions) {
     if (!OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS.includes(modelName)) {
       logger.warn(`Using unknown OpenAI chat model: ${modelName}`);
     }
     super(modelName, apiKey);
+    this.options = context || {};
   }
 
   // TODO(ian): support passing in `messages` directly
@@ -215,11 +232,17 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
     } catch (err) {
       messages = [{ role: 'user', content: prompt }];
     }
+
     const body = {
       model: this.modelName,
       messages: messages,
       max_tokens: parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
-      temperature: options?.temperature ?? parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      temperature:
+        options?.temperature ??
+        this.options.temperature ??
+        parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      functions: options?.functions || this.options.functions || undefined,
+      function_call: options?.function_call || this.options.function_call || undefined,
     };
     logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
 
@@ -246,8 +269,11 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
 
     logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
     try {
+      const message = data.choices[0].message;
+      const output =
+        message.content === null ? JSON.stringify(message.function_call) : message.content;
       return {
-        output: data.choices[0].message.content,
+        output,
         tokenUsage: cached
           ? { cached: data.usage.total_tokens }
           : {

package/src/providers.ts

@@ -1,20 +1,35 @@
 import path from 'node:path';
 
-import { ApiProvider } from './types';
+import { ApiProvider, ProviderConfig, ProviderId, RawProviderConfig } from './types';
 
 import { OpenAiCompletionProvider, OpenAiChatCompletionProvider } from './providers/openai';
 import { LocalAiCompletionProvider, LocalAiChatProvider } from './providers/localai';
 
-export async function loadApiProviders(providerPaths: string | string[]): Promise<ApiProvider[]> {
+export async function loadApiProviders(
+  providerPaths: ProviderId | ProviderId[] | RawProviderConfig[],
+): Promise<ApiProvider[]> {
   if (typeof providerPaths === 'string') {
     return [await loadApiProvider(providerPaths)];
   } else if (Array.isArray(providerPaths)) {
-    return Promise.all(providerPaths.map((provider) => loadApiProvider(provider)));
+    return Promise.all(
+      providerPaths.map((provider) => {
+        if (typeof provider === 'string') {
+          return loadApiProvider(provider);
+        } else {
+          const id = Object.keys(provider)[0];
+          const context = { ...provider[id], id };
+          return loadApiProvider(id, context);
+        }
+      }),
+    );
   }
   throw new Error('Invalid providers list');
 }
 
-export async function loadApiProvider(providerPath: string): Promise<ApiProvider> {
+export async function loadApiProvider(
+  providerPath: string,
+  context: ProviderConfig | undefined = undefined,
+): Promise<ApiProvider> {
   if (providerPath?.startsWith('openai:')) {
     // Load OpenAI module
     const options = providerPath.split(':');
@@ -22,13 +37,21 @@ export async function loadApiProvider(providerPath: string): Promise<ApiProvider
     const modelName = options[2];
 
     if (modelType === 'chat') {
-      return new OpenAiChatCompletionProvider(modelName || 'gpt-3.5-turbo');
+      return new OpenAiChatCompletionProvider(
+        modelName || 'gpt-3.5-turbo',
+        undefined,
+        context?.config,
+      );
     } else if (modelType === 'completion') {
-      return new OpenAiCompletionProvider(modelName || 'text-davinci-003');
+      return new OpenAiCompletionProvider(
+        modelName || 'text-davinci-003',
+        undefined,
+        context?.config,
+      );
     } else if (OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS.includes(modelType)) {
-      return new OpenAiChatCompletionProvider(modelType);
+      return new OpenAiChatCompletionProvider(modelType, undefined, context?.config);
     } else if (OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.includes(modelType)) {
-      return new OpenAiCompletionProvider(modelType);
+      return new OpenAiCompletionProvider(modelType, undefined, context?.config);
     } else {
       throw new Error(
         `Unknown OpenAI model type: ${modelType}. Use one of the following providers: openai:chat:<model name>, openai:completion:<model name>`,
@@ -52,7 +75,7 @@ export async function loadApiProvider(providerPath: string): Promise<ApiProvider
 
   // Load custom module
   const CustomApiProvider = (await import(path.join(process.cwd(), providerPath))).default;
-  return new CustomApiProvider();
+  return new CustomApiProvider(context);
 }
 
 export default {

package/src/table.ts

@@ -0,0 +1,41 @@
+import Table from 'cli-table3';
+import chalk from 'chalk';
+import type { EvaluateSummary } from './types';
+
+export function generateTable(summary: EvaluateSummary, tableCellMaxLength = 250, maxRows = 25) {
+  const maxWidth = process.stdout.columns ? process.stdout.columns - 10 : 120;
+  const head = summary.table.head;
+  const headLength = head.prompts.length + head.vars.length;
+  const table = new Table({
+    head: [...head.prompts, ...head.vars],
+    colWidths: Array(headLength).fill(Math.floor(maxWidth / headLength)),
+    wordWrap: true,
+    wrapOnWordBoundary: false,
+    style: {
+      head: ['blue', 'bold'],
+    },
+  });
+  // Skip first row (header) and add the rest. Color PASS/FAIL
+  for (const row of summary.table.body.slice(0, maxRows)) {
+    table.push([
+      ...row.vars,
+      ...row.outputs.map((col) => {
+        if (col.length > tableCellMaxLength) {
+          col = col.slice(0, tableCellMaxLength) + '...';
+        }
+        if (col.startsWith('[PASS]')) {
+          // color '[PASS]' green
+          return chalk.green.bold(col.slice(0, 6)) + col.slice(6);
+        } else if (col.startsWith('[FAIL]')) {
+          // color everything red up until '---'
+          return col
+            .split('---')
+            .map((c, idx) => (idx === 0 ? chalk.red.bold(c) : c))
+            .join('---');
+        }
+        return col;
+      }),
+    ]);
+  }
+  return table;
+}

package/src/telemetry.ts

@@ -0,0 +1,57 @@
+import packageJson from '../package.json';
+import { fetchWithTimeout } from './util';
+
+type TelemetryEvent = {
+  event: string;
+  packageVersion: string;
+  properties: Record<string, string | number>;
+};
+
+type TelemetryEventTypes = 'eval_ran' | 'assertion_used' | 'command_used';
+
+const TELEMETRY_ENDPOINT = 'https://api.promptfoo.dev/telemetry';
+
+const TELEMETRY_TIMEOUT_MS = 1000;
+
+export class Telemetry {
+  private events: TelemetryEvent[] = [];
+
+  get disabled() {
+    return process.env.PROMPTFOO_DISABLE_TELEMETRY === '1';
+  }
+
+  record(eventName: TelemetryEventTypes, properties: Record<string, string | number>): void {
+    if (!this.disabled) {
+      this.events.push({
+        event: eventName,
+        packageVersion: packageJson.version,
+        properties,
+      });
+    }
+  }
+
+  async send(): Promise<void> {
+    if (!this.disabled && this.events.length > 0) {
+      try {
+        const response = await fetchWithTimeout(
+          TELEMETRY_ENDPOINT,
+          {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json',
+            },
+            body: JSON.stringify(this.events),
+          },
+          TELEMETRY_TIMEOUT_MS,
+        );
+
+        if (response.ok) {
+          this.events = [];
+        }
+      } catch (err) {}
+    }
+  }
+}
+
+const telemetry = new Telemetry();
+export default telemetry;

package/src/types.ts

@@ -23,6 +23,11 @@ export interface CommandLineOptions {
   promptSuffix?: string;
 }
 
+export interface ProviderConfig {
+  id: ProviderId;
+  config?: any;
+}
+
 export interface ApiProvider {
   id: () => string;
   callApi: (prompt: string) => Promise<ProviderResponse>;
@@ -187,13 +192,17 @@ export interface TestSuite {
   defaultTest?: Partial<TestCase>;
 }
 
+export type ProviderId = string;
+
+export type RawProviderConfig = Record<ProviderId, Omit<ProviderConfig, 'id'>>;
+
 // TestSuiteConfig = Test Suite, but before everything is parsed and resolved.  Providers are just strings, prompts are filepaths, tests can be filepath or inline.
 export interface TestSuiteConfig {
   // Optional description of what your LLM is trying to do
   description?: string;
 
   // One or more LLM APIs to use, for example: openai:gpt-3.5-turbo, openai:gpt-4, localai:chat:vicuna
-  providers: string | string[];
+  providers: ProviderId | ProviderId[] | RawProviderConfig[];
 
   // One or more prompt files to load
   prompts: string | string[];

package/src/updates.ts

@@ -0,0 +1,37 @@
+import chalk from 'chalk';
+import semverGt from 'semver/functions/gt';
+
+import logger from './logger';
+import { fetchWithTimeout } from './util';
+import packageJson from '../package.json';
+
+const VERSION = packageJson.version;
+
+export async function getLatestVersion(packageName: string) {
+  const response = await fetchWithTimeout(`https://registry.npmjs.org/${packageName}`, {}, 1000);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch package information for ${packageName}`);
+  }
+  const data = await response.json();
+  return data['dist-tags'].latest;
+}
+
+export async function checkForUpdates(): Promise<boolean> {
+  const latestVersion = await getLatestVersion('promptfoo');
+  if (semverGt(latestVersion, VERSION)) {
+    const border = '='.repeat(process.stdout.columns - 10);
+    logger.info(
+      `\n${border}
+${chalk.yellow('⚠️')} The current version of promptfoo ${chalk.yellow(
+        VERSION,
+      )} is lower than the latest available version ${chalk.green(latestVersion)}.
+
+Please run ${chalk.green('npx promptfoo@latest')} or ${chalk.green(
+        'npm install -g promptfoo@latest',
+      )} to update.
+${border}\n`,
+    );
+    return true;
+  }
+  return false;
+}

package/src/util.ts

@@ -16,7 +16,15 @@ import { getDirectory } from './esm';
 
 import type { RequestInfo, RequestInit, Response } from 'node-fetch';
 
-import type { Assertion, CsvRow, EvaluateSummary, UnifiedConfig, TestCase, Prompt } from './types';
+import type {
+  Assertion,
+  CsvRow,
+  EvaluateSummary,
+  UnifiedConfig,
+  TestCase,
+  Prompt,
+  TestSuite,
+} from './types';
 import { assertionFromString } from './assertions';
 
 const PROMPT_DELIMITER = '---';
@@ -88,11 +96,22 @@ export function readPrompts(
       promptContents.push(...fileContents.map((content) => ({ raw: content, display: content })));
     } else {
       const fileContent = fs.readFileSync(promptPath, 'utf-8');
-      let display;
+
+      let display: string | undefined;
       if (inputType === PromptInputType.NAMED) {
         display = (promptPathOrGlobs as Record<string, string>)[promptPath];
       } else {
         display = fileContent.length > 200 ? promptPath : fileContent;
+
+        const ext = path.parse(promptPath).ext;
+        if (ext === '.jsonl') {
+          // Special case for JSONL file
+          const jsonLines = fileContent.split(/\r?\n/).filter((line) => line.length > 0);
+          for (const json of jsonLines) {
+            promptContents.push({ raw: json, display: json });
+          }
+          continue;
+        }
       }
       promptContents.push({ raw: fileContent, display });
     }
@@ -238,11 +257,22 @@ export function getLatestResultsPath(): string {
   return path.join(getConfigDirectoryPath(), 'output', 'latest.json');
 }
 
-export function writeLatestResults(results: EvaluateSummary) {
+export function writeLatestResults(results: EvaluateSummary, config: Partial<UnifiedConfig>) {
   const latestResultsPath = getLatestResultsPath();
   try {
     fs.mkdirSync(path.dirname(latestResultsPath), { recursive: true });
-    fs.writeFileSync(latestResultsPath, JSON.stringify(results, null, 2));
+    fs.writeFileSync(
+      latestResultsPath,
+      JSON.stringify(
+        {
+          version: 1,
+          config,
+          results,
+        },
+        null,
+        2,
+      ),
+    );
   } catch (err) {
     logger.error(`Failed to write latest results to ${latestResultsPath}:\n${err}`);
   }

package/src/web/client/package.json

@@ -15,6 +15,7 @@
     "@mui/icons-material": "^5.11.16",
     "@mui/material": "^5.13.0",
     "@tanstack/react-table": "^8.9.1",
+    "js-yaml": "^4.1.0",
     "react": "^18.2.0",
     "react-dnd": "^16.0.1",
     "react-dnd-html5-backend": "^16.0.1",

package/src/web/client/src/App.tsx

@@ -11,8 +11,9 @@ import { useStore } from './store.js';
 import './App.css';
 
 function App() {
-  const { table, setTable } = useStore();
+  const { table, setTable, setConfig } = useStore();
   const [loaded, setLoaded] = React.useState<boolean>(false);
+  const loadedFromApi = React.useRef(false);
 
   const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
   const [darkMode, setDarkMode] = React.useState(prefersDarkMode);
@@ -37,24 +38,47 @@ function App() {
   };
 
   React.useEffect(() => {
-    //const socket = SocketIOClient(`http://${window.location.host}`);
+    const fetchEvalData = async (id: string) => {
+      if (loadedFromApi.current) {
+        return;
+      }
+      loadedFromApi.current = true;
+      const response = await fetch(`https://api.promptfoo.dev/eval/${id}`);
+      const body = await response.json();
+      setTable(
+        body.data.results?.table ||
+          // Backwards compatibility with <= 0.12.0
+          body.data.table,
+      );
+      setConfig(body.data.config);
+      setLoaded(true);
+    };
+
     const socket = SocketIOClient(`http://localhost:15500`);
 
-    socket.on('init', (data) => {
-      console.log('Initialized socket connection');
-      setLoaded(true);
-      setTable(data.table);
-    });
+    const pathMatch = window.location.pathname.match(/\/eval\/([\w:-]+)/);
+    if (pathMatch) {
+      const id = pathMatch[1];
+      fetchEvalData(id);
+    } else {
+      socket.on('init', (data) => {
+        console.log('Initialized socket connection', data);
+        setLoaded(true);
+        setTable(data.results.table);
+        setConfig(data.config);
+      });
 
-    socket.on('update', (data) => {
-      console.log('Received data update');
-      setTable(data.table);
-    });
+      socket.on('update', (data) => {
+        console.log('Received data update', data);
+        setTable(data.results.table);
+        setConfig(data.config);
+      });
+    }
 
     return () => {
       socket.disconnect();
     };
-  }, [loaded, setTable]);
+  }, [setTable, setConfig]);
 
   return (
     <ThemeProvider theme={theme}>