promptfoo 0.5.1 → 0.6.0

package/src/main.ts

@@ -34,7 +34,7 @@ These prompts are nunjucks templates, so you can use logic like this:
 {% endif %}`;
   const dummyVars =
     'var1,var2,var3\nvalue1,value2,value3\nanother value1,another value2,another value3';
-  const dummyConfig = `export default {
+  const dummyConfig = `module.exports = {
   prompts: ['prompts.txt'],
   providers: ['openai:gpt-3.5-turbo'],
   vars: 'vars.csv',
@@ -79,6 +79,10 @@ async function main() {
     defaultConfig = (await import(pathJoin(process.cwd(), './promptfooconfig.js'))).default;
     logger.info('Loaded default config from promptfooconfig.js');
   }
+  if (existsSync('promptfooconfig.json')) {
+    defaultConfig = JSON.parse(readFileSync('promptfooconfig.json', 'utf-8'));
+    logger.info('Loaded default config from promptfooconfig.json');
+  }
 
   const program = new Command();
 
@@ -143,10 +147,24 @@ async function main() {
       'Truncate console table cells to this length',
       '250',
     )
+    .option(
+      '--suggest-prompts <number>',
+      'Generate N new prompts and append them to the prompt list',
+    )
+    .option(
+      '--prompt-prefix <path>',
+      'This prefix is prepended to every prompt',
+      defaultConfig.promptPrefix,
+    )
+    .option(
+      '--prompt-suffix <path>',
+      'This suffix is append to every prompt',
+      defaultConfig.promptSuffix,
+    )
     .option('--no-write', 'Do not write results to promptfoo directory')
     .option('--grader', 'Model that will grade outputs', defaultConfig.grader)
     .option('--verbose', 'Show debug logs', defaultConfig.verbose)
-    .option('--view', 'View in browser ui')
+    .option('--view [port]', 'View in browser ui')
     .action(async (cmdObj: CommandLineOptions & Command) => {
       if (cmdObj.verbose) {
         setLogLevel('debug');
@@ -184,6 +202,10 @@ async function main() {
         providers,
         showProgressBar: true,
         maxConcurrency: !isNaN(maxConcurrency) && maxConcurrency > 0 ? maxConcurrency : undefined,
+        prompt: {
+          prefix: cmdObj.promptPrefix,
+          suffix: cmdObj.promptSuffix,
+        },
         ...config,
       };
 
@@ -192,6 +214,9 @@ async function main() {
           provider: await loadApiProvider(cmdObj.grader),
         };
       }
+      if (cmdObj.generateSuggestions) {
+        options.prompt!.generateSuggestions = true;
+      }
 
       const summary = await evaluate(options);
 
@@ -252,7 +277,7 @@ async function main() {
       logger.info('Done.');
 
       if (cmdObj.view) {
-        init(15500);
+        init(parseInt(cmdObj.view, 10) || 15500);
       }
     });
 

package/src/prompts.ts

@@ -18,3 +18,12 @@ Rubric: Does not speak like a pirate
     content: 'Content: {{ content }}\nRubric: {{ rubric }}',
   },
 ]);
+
+export const SUGGEST_PROMPTS_SYSTEM_MESSAGE = {
+  role: 'system',
+  content: `You're helping a scientist who is tuning a prompt for a large language model.  You will receive messages, and each message is a full prompt.  Generate a candidate variation of the given prompt.  This variation will be tested for quality in order to select a winner.
+
+Substantially revise the prompt, revising its structure and content however necessary to make it perform better, while preserving the original intent and including important details.
+
+Your output is going to be copied directly into the program. It should contain the prompt ONLY`,
+};

package/src/providers/openai.ts

@@ -12,6 +12,10 @@ const embeddingsCache = new LRUCache<string, ProviderEmbeddingResponse>({
   max: 1000,
 });
 
+interface OpenAiCompletionOptions {
+  temperature: number;
+}
+
 class OpenAiGenericProvider implements ApiProvider {
   modelName: string;
   apiKey?: string;
@@ -34,7 +38,7 @@ class OpenAiGenericProvider implements ApiProvider {
   }
 
   // @ts-ignore: Prompt is not used in this implementation
-  async callApi(prompt: string): Promise<ProviderResponse> {
+  async callApi(prompt: string, options?: OpenAiCompletionOptions): Promise<ProviderResponse> {
     throw new Error('Not implemented');
   }
 }
@@ -110,8 +114,6 @@ export class OpenAiEmbeddingProvider extends OpenAiGenericProvider {
   }
 }
 
-export const DefaultEmbeddingProvider = new OpenAiEmbeddingProvider('text-embedding-ada-002');
-
 export class OpenAiCompletionProvider extends OpenAiGenericProvider {
   static OPENAI_COMPLETION_MODELS = [
     'text-davinci-003',
@@ -126,20 +128,20 @@ export class OpenAiCompletionProvider extends OpenAiGenericProvider {
       logger.warn(`Using unknown OpenAI completion model: ${modelName}`);
     }
     super(modelName, apiKey);
+  }
 
+  async callApi(prompt: string, options?: OpenAiCompletionOptions): Promise<ProviderResponse> {
     if (!this.apiKey) {
       throw new Error(
         'OpenAI API key is not set. Set OPENAI_API_KEY environment variable or pass it as an argument to the constructor.',
       );
     }
-  }
 
-  async callApi(prompt: string): Promise<ProviderResponse> {
     const body = {
       model: this.modelName,
       prompt,
       max_tokens: process.env.OPENAI_MAX_TOKENS || 1024,
-      temperature: process.env.OPENAI_TEMPERATURE || 0,
+      temperature: options?.temperature ?? (process.env.OPENAI_MAX_TEMPERATURE || 0),
       stop: process.env.OPENAI_STOP ? JSON.parse(process.env.OPENAI_STOP) : undefined,
     };
     logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
@@ -197,15 +199,16 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
       logger.warn(`Using unknown OpenAI chat model: ${modelName}`);
     }
     super(modelName, apiKey);
+  }
 
+  // TODO(ian): support passing in `messages` directly
+  async callApi(prompt: string, options?: OpenAiCompletionOptions): Promise<ProviderResponse> {
     if (!this.apiKey) {
       throw new Error(
         'OpenAI API key is not set. Set OPENAI_API_KEY environment variable or pass it as an argument to the constructor.',
       );
     }
-  }
 
-  async callApi(prompt: string): Promise<ProviderResponse> {
     let messages: { role: string; content: string }[];
     try {
       // User can specify `messages` payload as JSON, or we'll just put the
@@ -218,7 +221,7 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
       model: this.modelName,
       messages: messages,
       max_tokens: process.env.OPENAI_MAX_TOKENS || 1024,
-      temperature: process.env.OPENAI_MAX_TEMPERATURE || 0,
+      temperature: options?.temperature ?? (process.env.OPENAI_MAX_TEMPERATURE || 0),
     };
     logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
 
@@ -260,3 +263,7 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
     }
   }
 }
+
+export const DefaultEmbeddingProvider = new OpenAiEmbeddingProvider('text-embedding-ada-002');
+export const DefaultGradingProvider = new OpenAiChatCompletionProvider('gpt-4');
+export const DefaultSuggestionsProvider = new OpenAiChatCompletionProvider('gpt-4');

package/src/providers/shared.ts

@@ -1,3 +1,3 @@
 export const REQUEST_TIMEOUT_MS = process.env.REQUEST_TIMEOUT_MS
   ? parseInt(process.env.REQUEST_TIMEOUT_MS, 10)
-  : 60_000;
+  : 300_000;

package/src/providers.ts

@@ -45,3 +45,11 @@ export async function loadApiProvider(providerPath: string): Promise<ApiProvider
   const CustomApiProvider = (await import(path.join(process.cwd(), providerPath))).default;
   return new CustomApiProvider();
 }
+
+export default {
+  OpenAiCompletionProvider,
+  OpenAiChatCompletionProvider,
+  LocalAiCompletionProvider,
+  LocalAiChatProvider,
+  loadApiProvider,
+};

package/src/suggestions.ts

@@ -0,0 +1,63 @@
+import { SUGGEST_PROMPTS_SYSTEM_MESSAGE } from './prompts';
+import { DefaultSuggestionsProvider } from './providers/openai';
+
+import type { TokenUsage } from './types';
+
+const DEFAULT_TEMPERATURE = 0.9;
+
+interface GeneratePromptsOutput {
+  prompts?: string[];
+  error?: string;
+  tokensUsed: TokenUsage;
+}
+
+export async function generatePrompts(prompt: string, num: number): Promise<GeneratePromptsOutput> {
+  const provider = DefaultSuggestionsProvider;
+
+  const resp = await provider.callApi(
+    JSON.stringify([
+      SUGGEST_PROMPTS_SYSTEM_MESSAGE,
+      {
+        role: 'user',
+        content: 'Generate a variant for the following prompt:',
+      },
+      {
+        role: 'user',
+        content: prompt,
+      },
+    ]),
+    {
+      temperature: DEFAULT_TEMPERATURE,
+    },
+  );
+  if (resp.error || !resp.output) {
+    return {
+      error: resp.error || 'Unknown error',
+      tokensUsed: {
+        total: resp.tokenUsage?.total || 0,
+        prompt: resp.tokenUsage?.prompt || 0,
+        completion: resp.tokenUsage?.completion || 0,
+      },
+    };
+  }
+
+  try {
+    return {
+      prompts: [resp.output],
+      tokensUsed: {
+        total: resp.tokenUsage?.total || 0,
+        prompt: resp.tokenUsage?.prompt || 0,
+        completion: resp.tokenUsage?.completion || 0,
+      },
+    };
+  } catch (err) {
+    return {
+      error: `Output is not valid JSON: ${resp.output}`,
+      tokensUsed: {
+        total: resp.tokenUsage?.total || 0,
+        prompt: resp.tokenUsage?.prompt || 0,
+        completion: resp.tokenUsage?.completion || 0,
+      },
+    };
+  }
+}

package/src/types.ts

@@ -7,9 +7,13 @@ export interface CommandLineOptions {
   verbose?: boolean;
   maxConcurrency?: string;
   grader?: string;
-  view?: boolean;
+  view?: string;
   noWrite?: boolean;
   tableCellMaxLength?: string;
+
+  generateSuggestions?: boolean;
+  promptPrefix?: string;
+  promptSuffix?: string;
 }
 
 export interface ApiProvider {
@@ -43,7 +47,13 @@ export type VarMapping = Record<string, string>;
 
 export interface GradingConfig {
   prompt?: string;
-  provider: ApiProvider;
+  provider?: string | ApiProvider;
+}
+
+export interface PromptConfig {
+  prefix?: string;
+  suffix?: string;
+  generateSuggestions?: boolean;
 }
 
 export interface EvaluateOptions {
@@ -55,6 +65,8 @@ export interface EvaluateOptions {
   showProgressBar?: boolean;
 
   grading?: GradingConfig;
+
+  prompt?: PromptConfig;
 }
 
 export interface Prompt {

package/src/util.ts

@@ -5,6 +5,7 @@ import * as os from 'node:os';
 import fetch from 'node-fetch';
 import yaml from 'js-yaml';
 import nunjucks from 'nunjucks';
+import { globSync } from 'glob';
 import { parse as parsePath } from 'path';
 import { CsvRow } from './types.js';
 import { parse as parseCsv } from 'csv-parse/sync';
@@ -27,8 +28,24 @@ function parseJson(json: string): any | undefined {
   }
 }
 
-export function readPrompts(promptPaths: string[]): string[] {
-  let promptContents = promptPaths.map((path) => fs.readFileSync(path, 'utf-8'));
+export function readPrompts(promptPathsOrGlobs: string[]): string[] {
+  const promptPaths = promptPathsOrGlobs.flatMap((pathOrGlob) => globSync(pathOrGlob));
+  let promptContents: string[] = [];
+
+  for (const promptPath of promptPaths) {
+    const stat = fs.statSync(promptPath);
+    if (stat.isDirectory()) {
+      const filesInDirectory = fs.readdirSync(promptPath);
+      const fileContents = filesInDirectory.map((fileName) =>
+        fs.readFileSync(path.join(promptPath, fileName), 'utf-8'),
+      );
+      promptContents.push(...fileContents);
+    } else {
+      const fileContent = fs.readFileSync(promptPath, 'utf-8');
+      promptContents.push(fileContent);
+    }
+  }
+
   if (promptContents.length === 1) {
     promptContents = promptContents[0].split(PROMPT_DELIMITER).map((p) => p.trim());
   }
@@ -65,8 +82,12 @@ export function writeOutput(outputPath: string, summary: EvaluateSummary): void
     fs.writeFileSync(outputPath, yaml.dump(summary));
   } else if (outputExtension === 'html') {
     const template = fs.readFileSync(`${getDirectory()}/tableOutput.html`, 'utf-8');
+    const table = [
+      [...summary.table.head.prompts, ...summary.table.head.vars],
+      ...summary.table.body.map((row) => [...row.outputs, ...row.vars]),
+    ];
     const htmlOutput = nunjucks.renderString(template, {
-      table: summary.table,
+      table,
       results: summary.results,
     });
     fs.writeFileSync(outputPath, htmlOutput);

package/src/web/client/package.json

@@ -12,6 +12,7 @@
   "dependencies": {
     "@emotion/react": "^11.11.0",
     "@emotion/styled": "^11.11.0",
+    "@mui/icons-material": "^5.11.16",
     "@mui/material": "^5.13.0",
     "@tanstack/react-table": "^8.9.1",
     "react": "^18.2.0",

package/src/web/client/src/App.css

@@ -0,0 +1,4 @@
+body {
+  background-color: var(--background-color);
+  color: var(--text-color);
+}

package/src/web/client/src/App.tsx

@@ -1,5 +1,7 @@
 import * as React from 'react';
 
+import useMediaQuery from '@mui/material/useMediaQuery';
+import { ThemeProvider, createTheme } from '@mui/material/styles';
 import { io as SocketIOClient } from 'socket.io-client';
 
 import ResultsView from './ResultsView.js';
@@ -12,9 +14,31 @@ function App() {
   const { table, setTable } = useStore();
   const [loaded, setLoaded] = React.useState<boolean>(false);
 
+  const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
+  const [darkMode, setDarkMode] = React.useState(prefersDarkMode);
+
+  const theme = React.useMemo(
+    () =>
+      createTheme({
+        palette: {
+          mode: darkMode ? 'dark' : 'light',
+        },
+      }),
+    [darkMode],
+  );
+
+  const toggleDarkMode = () => {
+    setDarkMode(!darkMode);
+    if (!darkMode) {
+      document.documentElement.setAttribute('data-theme', 'dark');
+    } else {
+      document.documentElement.removeAttribute('data-theme');
+    }
+  };
+
   React.useEffect(() => {
-    const socket = SocketIOClient(`http://${window.location.host}`);
-    //const socket = SocketIOClient(`http://localhost:15500`);
+    //const socket = SocketIOClient(`http://${window.location.host}`);
+    const socket = SocketIOClient(`http://localhost:15500`);
 
     socket.on('init', (data) => {
       console.log('Initialized socket connection');
@@ -33,10 +57,10 @@ function App() {
   }, [loaded, setTable]);
 
   return (
-    <>
-      <NavBar />
+    <ThemeProvider theme={theme}>
+      <NavBar darkMode={darkMode} onToggleDarkMode={toggleDarkMode} />
       {loaded && table ? <ResultsView /> : <div>Loading...</div>}
-    </>
+    </ThemeProvider>
   );
 }
 

package/src/web/client/src/Logo.css

@@ -10,4 +10,9 @@
 
 .logo span {
   margin-bottom: 6px;
+  color: var(--text-color);
+}
+
+[data-theme='dark'] .logo img {
+  filter: invert(1);
 }

package/src/web/client/src/NavBar.css

@@ -1,3 +1,21 @@
 nav {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
   margin-bottom: 1rem;
+  color: var(--text-color);
+}
+
+.dark-mode-toggle {
+  background-color: transparent;
+  border: none;
+  color: var(--text-color);
+  cursor: pointer;
+  font-size: 16px;
+  padding: 8px;
+  transition: color 0.3s;
+}
+
+.dark-mode-toggle:hover {
+  color: var(--pass-color);
 }

package/src/web/client/src/NavBar.tsx

@@ -1,11 +1,22 @@
 import Logo from './Logo';
 
+import DarkModeIcon from '@mui/icons-material/DarkMode';
+import LightModeIcon from '@mui/icons-material/LightMode';
+
 import './NavBar.css';
 
-export default function NavBar() {
+interface NavbarProps {
+  darkMode: boolean;
+  onToggleDarkMode: () => void;
+}
+
+export default function NavBar({ darkMode, onToggleDarkMode }: NavbarProps) {
   return (
     <nav>
       <Logo />
+      <div className="dark-mode-toggle" onClick={onToggleDarkMode}>
+        {darkMode ? <DarkModeIcon /> : <LightModeIcon />}
+      </div>
     </nav>
   );
 }

package/src/web/client/src/index.css

@@ -30,6 +30,16 @@
   }
 }
 
+[data-theme='dark'] {
+  --background-color: #1a1a1a;
+  --text-color: #f0f0f0;
+  --border-color: #444444;
+  --table-border-color: #444444;
+  --pass-color: #4caf50;
+  --fail-color: #f44336;
+  --smalltext-color: #888888;
+}
+
 html {
   font-size: calc(14px + (18 - 14) * ((100vw - 300px) / (1600 - 300)));
 }

package/src/web/server.ts

@@ -4,9 +4,9 @@ import readline from 'node:readline';
 import http from 'node:http';
 
 import debounce from 'debounce';
-import open from 'open';
 import express from 'express';
 import cors from 'cors';
+import opener from 'opener';
 import { Server as SocketIOServer } from 'socket.io';
 
 import promptfoo from '../index.js';
@@ -83,7 +83,7 @@ export function init(port = 15500) {
     rl.question('Do you want to open the browser to the URL? (y/N): ', async (answer) => {
       if (answer.toLowerCase().startsWith('y')) {
         try {
-          await open(url);
+          await opener(url);
           logger.info(`Opening browser to: ${url}`);
         } catch (err) {
           logger.error(`Failed to open browser: ${String(err)}`);

package/dist/web/client/assets/index-710f1308.css

@@ -1 +0,0 @@
-:root{font-family:system-ui,Avenir,Helvetica,Arial,sans-serif;font-synthesis:none;text-rendering:optimizeLegibility;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;-webkit-text-size-adjust:100%;--background-color: #ffffff;--text-color: #404040;--border-color: lightgray;--table-border-color: lightgray;--pass-color: green;--fail-color: #ad0000;--smalltext-color: gray}@media (prefers-color-scheme: dark){:root{--background-color: #1a1a1a;--text-color: #f0f0f0;--border-color: #444444;--table-border-color: #444444;--pass-color: #4caf50;--fail-color: #f44336;--smalltext-color: #888888}}html{font-size:calc(14px + (18 - 14) * ((100vw - 300px) / (1600 - 300)))}*{box-sizing:border-box}html{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji",Segoe UI Symbol;font-size:16px;background-color:var(--background-color);color:var(--text-color)}table,.divTable{border:1px solid var(--table-border-color);border-collapse:collapse;width:100%;margin:1rem 0;box-shadow:0 2px 4px #0000001a}.tr{display:flex}tr,.tr{width:fit-content}tr:hover,.tr:hover{background-color:#0000000d}th,.th,td,.td{position:relative;box-shadow:inset 0 0 0 1px var(--border-color);word-break:break-all;vertical-align:top;padding:1.5rem}th,.th{padding:1rem;position:relative;text-align:center;font-weight:semi-bold}tr .cell-rating{visibility:hidden;position:absolute;bottom:1.25rem;right:-1rem;line-height:0;font-size:1.75rem}tr:hover .cell-rating{visibility:visible}tr .cell-rating .rating{cursor:pointer;margin-right:1rem}th .smalltext{visibility:hidden;font-weight:400;font-size:.75rem;color:var(--smalltext-color)}th:hover .smalltext{visibility:visible}td .status{margin-bottom:.5rem;font-weight:700}td .pass{color:var(--pass-color)}td .fail{color:var(--fail-color)}.resizer{position:absolute;right:0;top:0;height:100%;width:5px;cursor:col-resize;user-select:none;touch-action:none;background:var(--text-color);opacity:.5}.resizer.isResizing{background:var(--text-color);opacity:1}@media (hover: hover){.resizer{opacity:0}*:hover>.resizer{opacity:1}}.logo{display:flex;align-items:center;gap:4px}.logo img{width:30px}.logo span{margin-bottom:6px}nav{margin-bottom:1rem}